diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 05fb224..0000000 --- a/.dockerignore +++ /dev/null @@ -1,17 +0,0 @@ -# Version and commit come in as build args, so the build doesn't need .git. -.git -.github - -# Local build output and runtime data -/proxy -dist/ -build/ -cache/ -*.db -*.tar.gz - -# Local config and secrets -.env -.env.local -proxy.cfg -config.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d2e324..700f28e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,17 +13,18 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] + go-version: ['1.25'] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: - go-version-file: go.mod + go-version: ${{ matrix.go-version }} - name: Build run: go build -v ./... @@ -31,92 +32,17 @@ jobs: - name: Test run: go test -v -race ./... - postgres: - name: test (postgres) - runs-on: ubuntu-latest - services: - postgres: - image: postgres:17@sha256:b8faeff7712663c2773a190181e1246e37e364b887611c04658c28b44e2408fe - env: - POSTGRES_PASSWORD: test - POSTGRES_DB: proxy - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres -d proxy" - --health-interval 2s - --health-timeout 5s - --health-retries 30 - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: go.mod - - # The packages with tests that need PROXY_DATABASE_URL. -p 1 because - # their tests drop and recreate tables in the one shared database. - - name: Test - env: - PROXY_DATABASE_URL: postgres://postgres:test@localhost:5432/proxy?sslmode=disable - run: go test -v -race -p 1 ./internal/database/... ./internal/handler/... ./internal/server/... - lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: - go-version-file: go.mod - - - name: Install golangci-lint - run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.15.0 + go-version: '1.25' - name: golangci-lint - run: golangci-lint run ./... - - helm: - name: Helm chart - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - with: - version: v3.18.6 - - - name: Lint chart - run: helm lint deploy/charts/proxy - - - name: Render chart variants - run: | - set -euo pipefail - helm template proxy deploy/charts/proxy >/dev/null - helm template proxy deploy/charts/proxy \ - --set persistence.enabled=false \ - --set config.existingConfigMap=proxy-config \ - --set ingress.enabled=true \ - --set 'ingress.hosts[0].host=proxy.example.com' \ - --set 'ingress.hosts[0].paths[0].path=/' \ - --set 'ingress.hosts[0].paths[0].pathType=Prefix' \ - >/dev/null - helm template proxy deploy/charts/proxy \ - --set 'extraVolumes[0].name=corp-ca' \ - --set 'extraVolumes[0].configMap.name=corp-ca' \ - --set 'extraVolumeMounts[0].name=corp-ca' \ - --set 'extraVolumeMounts[0].mountPath=/etc/proxy/ca' \ - --set 'extraVolumeMounts[0].readOnly=true' \ - --set 'extraInitContainers[0].name=wait-for-db' \ - --set 'extraInitContainers[0].image=busybox:1.37' \ - --set 'extraInitContainers[0].command={sh,-c,exit 0}' \ - >/dev/null + run: go tool golangci-lint run ./... diff --git a/.github/workflows/monthly-release.yml b/.github/workflows/monthly-release.yml deleted file mode 100644 index b0da481..0000000 --- a/.github/workflows/monthly-release.yml +++ /dev/null @@ -1,137 +0,0 @@ -name: Monthly release - -# Cuts a release on the 1st of each month when main is green and something -# has merged since the last release: a patch release, or a new minor version -# if a pull request labelled "breaking" was merged (a new major version from -# 1.0 on). A breaking release is created as a draft, and its images and chart -# are published only when someone publishes the draft, after writing the -# upgrade notes. Scheduled runs only report what they would do until the -# repository variable MONTHLY_RELEASE is "true". - -on: - schedule: - - cron: "0 6 1 * *" - workflow_dispatch: - inputs: - dry_run: - description: Only report what would be released - type: boolean - default: true - -permissions: {} - -concurrency: - group: monthly-release - -jobs: - release: - runs-on: ubuntu-latest - permissions: - contents: write - actions: write - checks: read - pull-requests: read - env: - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: main - fetch-depth: 0 - persist-credentials: false - - - name: Decide whether to release - id: plan - env: - EVENT: ${{ github.event_name }} - INPUT_DRY_RUN: ${{ inputs.dry_run }} - ENABLED: ${{ vars.MONTHLY_RELEASE }} - run: | - set -euo pipefail - if [ "$EVENT" = workflow_dispatch ]; then - dry_run=$INPUT_DRY_RUN - elif [ "$ENABLED" = true ]; then - dry_run=false - else - dry_run=true - fi - echo "dry_run=$dry_run" >> "$GITHUB_OUTPUT" - - sha=$(git rev-parse HEAD) - last=$(git describe --tags --abbrev=0 --match 'v[0-9]*.[0-9]*.[0-9]*' --exclude '*-*') - # A breaking release waits as a draft for its upgrade notes. Building - # on its tag would publish those changes without them. Older tags - # were released by hand and may have no release at all. - draft=$(gh release view "$last" -R "$REPO" --json isDraft --jq .isDraft 2>/dev/null || echo false) - if [ "$draft" = true ]; then - echo "::warning::$last is still a draft, publish it first" - exit 0 - fi - merged=$(git rev-list --count "$last..$sha") - if [ "$merged" -eq 0 ]; then - echo "Nothing merged since $last" - exit 0 - fi - - # Every check on main's HEAD must have finished green. This run - # shows up as a check on the same commit, so it is left out. - bad=$(gh api --paginate "repos/$REPO/commits/$sha/check-runs?per_page=100" \ - --jq ".check_runs[] - | select(.html_url | contains(\"/actions/runs/$GITHUB_RUN_ID/\") | not) - | select(.status != \"completed\" - or (.conclusion | IN(\"success\", \"skipped\", \"neutral\") | not)) - | .name") - if [ -n "$bad" ]; then - echo "::warning::main ($sha) is not green, skipping: $(echo "$bad" | sort -u | paste -sd, -)" - exit 0 - fi - - since=$(git log -1 --format=%cI "$last") - breaking=$(gh pr list -R "$REPO" --state merged --base main --limit 1000 \ - --search "merged:>$since label:breaking" --json number --jq '.[].number') - - IFS=. read -r major minor patch <<<"${last#v}" - if [ -z "$breaking" ]; then - tag="v$major.$minor.$((patch + 1))" - else - echo "Breaking changes merged since $last: #$(echo "$breaking" | sort -n | paste -sd, - | sed 's/,/, #/g')" - if [ "$major" -eq 0 ]; then - tag="v0.$((minor + 1)).0" - else - tag="v$((major + 1)).0.0" - fi - fi - echo "$merged commits since $last, next release $tag at $sha" - echo "tag=$tag" >> "$GITHUB_OUTPUT" - echo "sha=$sha" >> "$GITHUB_OUTPUT" - echo "draft=$([ -n "$breaking" ] && echo true || echo false)" >> "$GITHUB_OUTPUT" - - # A tag pushed with GITHUB_TOKEN doesn't start other workflows, so the - # release workflows are dispatched on the new tag instead. - - name: Tag and start the release workflows - if: steps.plan.outputs.tag != '' - env: - TAG: ${{ steps.plan.outputs.tag }} - SHA: ${{ steps.plan.outputs.sha }} - DRY_RUN: ${{ steps.plan.outputs.dry_run }} - DRAFT: ${{ steps.plan.outputs.draft }} - run: | - set -euo pipefail - if [ "$DRAFT" = true ]; then - what="start release.yml for a draft release" - else - what="start release.yml and publish.yml" - fi - if [ "$DRY_RUN" = true ]; then - echo "Dry run: would tag $TAG at $SHA and $what" - exit 0 - fi - gh api "repos/$REPO/git/refs" -f ref="refs/tags/$TAG" -f sha="$SHA" >/dev/null - gh workflow run release.yml -R "$REPO" --ref "$TAG" -f draft="$DRAFT" - if [ "$DRAFT" = true ]; then - echo "::notice::$TAG is a draft release: add the upgrade notes and publish it to push the images and chart" - else - gh workflow run publish.yml -R "$REPO" --ref "$TAG" - fi - echo "Tagged $TAG at $SHA" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e325a8a..93b0a1e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,10 +5,6 @@ on: push: tags: - "v*" - # A draft release made by monthly-release.yml is published by hand. Releases - # published with GITHUB_TOKEN don't start this. - release: - types: [published] permissions: {} @@ -22,23 +18,14 @@ jobs: permissions: packages: write contents: read - id-token: write steps: - name: Check out the repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 with: persist-credentials: false - - name: Set up QEMU - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 - with: - platforms: linux/amd64,linux/arm64 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - - name: Log in to the Container registry - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee with: registry: ghcr.io username: ${{ github.actor }} @@ -46,123 +33,14 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 + uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 with: images: ghcr.io/${{ github.repository }} - - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - - name: Build and push Docker image - id: build - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf with: context: . - platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ steps.meta.outputs.version }} - COMMIT=${{ github.sha }} - provenance: mode=max - sbom: true - - - name: Sign image by digest - env: - DIGEST: ${{ steps.build.outputs.digest }} - IMAGE: ghcr.io/${{ github.repository }} - run: | - set -euo pipefail - [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] - cosign sign --yes "${IMAGE}@${DIGEST}" - - - name: Verify BuildKit attestations and extract SPDX predicates - env: - DIGEST: ${{ steps.build.outputs.digest }} - IMAGE: ghcr.io/${{ github.repository }} - run: | - set -euo pipefail - reference="${IMAGE}@${DIGEST}" - docker buildx imagetools inspect "$reference" --format '{{ json .Provenance }}' > provenance.json - docker buildx imagetools inspect "$reference" --format '{{ json .SBOM }}' > sbom.json - - for platform in linux/amd64 linux/arm64; do - jq -e --arg p "$platform" '.[$p].SLSA | type == "object" and length > 0' \ - provenance.json >/dev/null - jq -e --arg p "$platform" '.[$p].SPDX' sbom.json > "sbom-${platform//\//-}.spdx.json" - done - - - name: Attest platform SBOMs by digest - env: - DIGEST: ${{ steps.build.outputs.digest }} - IMAGE: ghcr.io/${{ github.repository }} - run: | - set -euo pipefail - [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] - reference="${IMAGE}@${DIGEST}" - for predicate in sbom-linux-amd64.spdx.json sbom-linux-arm64.spdx.json; do - cosign attest --yes --type spdxjson --predicate "$predicate" "$reference" - done - - publish_chart: - name: Push Helm chart to GHCR - if: github.ref_type == 'tag' - needs: push_to_registry - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Check out the repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - persist-credentials: false - ref: ${{ github.sha }} - - - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - with: - version: v3.18.6 - - - name: Validate and normalize release version - id: version - env: - TAG: ${{ github.ref_name }} - run: | - set -euo pipefail - semver='^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*))?$' - [[ "$TAG" =~ $semver ]] || { - echo "Tag must be strict SemVer of the form vMAJOR.MINOR.PATCH[-PRERELEASE]: $TAG" >&2 - exit 1 - } - version="${TAG#v}" - [[ "$version" != "0.0.0" ]] || { - echo "0.0.0 is a development placeholder and must not be published" >&2 - exit 1 - } - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Log in to GHCR - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: printf '%s' "$GH_TOKEN" | helm registry login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - - - name: Lint and package chart - env: - VERSION: ${{ steps.version.outputs.version }} - run: | - set -euo pipefail - helm lint deploy/charts/proxy - mkdir -p build - helm package \ - --destination build \ - --version "$VERSION" \ - --app-version "$VERSION" \ - deploy/charts/proxy - metadata="$(helm show chart "build/proxy-${VERSION}.tgz")" - [[ "$(awk '$1 == "version:" {print $2}' <<<"$metadata")" == "$VERSION" ]] - [[ "$(awk '$1 == "appVersion:" {gsub(/\"/, "", $2); print $2}' <<<"$metadata")" == "$VERSION" ]] - - - name: Push chart - env: - VERSION: ${{ steps.version.outputs.version }} - run: helm push "build/proxy-${VERSION}.tgz" oci://ghcr.io/git-pkgs/charts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 491d6ec..f7833fb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,13 +4,6 @@ on: push: tags: - "v*" - # Started by monthly-release.yml on the tag it creates. - workflow_dispatch: - inputs: - draft: - description: Create the release as a draft - type: boolean - default: false permissions: contents: write @@ -18,11 +11,10 @@ permissions: jobs: release: - if: github.ref_type == 'tag' runs-on: ubuntu-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 persist-credentials: false @@ -30,14 +22,14 @@ jobs: - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod cache: false - - uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7.3.0 + - uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2 with: version: "~> v2" - args: release --clean${{ inputs.draft && ' --draft' || '' }} + args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index 4112a99..f947c7e 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -12,17 +12,17 @@ jobs: swagger: runs-on: ubuntu-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: - go-version-file: go.mod + go-version: '1.25' - name: Install swag - run: go install github.com/swaggo/swag/cmd/swag@v1.16.6 + run: go install github.com/swaggo/swag/cmd/swag@latest - name: Generate swagger run: go generate ./internal/server diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 00041a6..cce6e4f 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -5,22 +5,14 @@ on: branches: - main paths: - - '.github/**' - - '**/action.yml' - - '**/action.yaml' - - 'zizmor.yml' - - 'zizmor.yaml' + - '.github/workflows/**' pull_request: + branches: + - main paths: - - '.github/**' - - '**/action.yml' - - '**/action.yaml' - - 'zizmor.yml' - - 'zizmor.yaml' + - '.github/workflows/**' workflow_dispatch: -permissions: {} - jobs: zizmor: runs-on: ubuntu-latest @@ -29,9 +21,9 @@ jobs: security-events: write steps: - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0 + uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 diff --git a/.gitignore b/.gitignore index 86f77d1..9016252 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ *.dll *.so *.dylib -/proxy +proxy # Test binary, built with `go test -c` *.test @@ -43,4 +43,4 @@ cache/* # Debug files __debug_bin -debug +debug \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 57c0f69..9d4b957 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,7 +26,3 @@ linters: - goconst - dupl - mnd - -formatters: - enable: - - gofmt diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 9aec8f9..e5881db 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -15,7 +15,6 @@ builds: - linux - darwin - windows - - freebsd goarch: - amd64 - arm64 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6674b8c..88ad1cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,7 +39,7 @@ proxy/ │ │ └── queries.go # CRUD operations │ ├── storage/ # Artifact file storage │ │ ├── storage.go # Storage interface -│ │ └── blob.go # gocloud.dev/blob backends (file, S3, Azure) +│ │ └── filesystem.go # Local filesystem impl │ ├── upstream/ # Upstream registry clients │ │ ├── fetcher.go # HTTP artifact fetching │ │ └── resolver.go # Download URL resolution @@ -72,7 +72,7 @@ Key types: ### `internal/storage` -Artifact file storage abstraction backed by `gocloud.dev/blob`. Supports local filesystem (`file://`), S3 (`s3://`), and Azure (`azblob://`) URLs. +Artifact file storage abstraction. Currently implements local filesystem storage. Designed to allow future backends (S3, GCS). Interface: ```go @@ -160,7 +160,6 @@ mux.Handle("/newregistry/", http.StripPrefix("/newregistry", newHandler.Routes() - Keep functions short and focused - Write tests for new functionality - Document exported types and functions -- A new Prometheus metric needs a tile on `/ui/analytics` and an entry in `metricSurface` (`internal/server/analytics_coverage_test.go`). The page is meant to be a complete view of `/metrics`, so `TestEveryMetricIsSurfaced` fails until both exist. ## Testing @@ -212,24 +211,6 @@ Add PyPI registry support - Add tests for wheel and sdist downloads ``` -## Reviewing and Merging - -Maintainers are the members of the proxy team in the git-pkgs org. Any maintainer can review and merge. A pull request needs green CI and an approval from a maintainer other than its author. A "request changes" review from any maintainer blocks it until that reviewer approves or withdraws the review. - -Using AI to write or review is fine, but whoever approves a pull request should understand it and stand behind it. If it touches an ecosystem you don't use, you can ask someone who does to look before you approve. - -Open an issue before a bigger change: anything that changes behaviour users rely on, new configuration that would be hard to remove, or a change that fans out into other git-pkgs modules. If no maintainer objects within a few days, open the pull request; it's reviewed like any other. Objections are worked out in the issue, with @andrew breaking ties for now. Where possible, deprecate before removing: an old configuration key keeps working, with a warning, in at least one release before the one that removes it. - -## Releases - -A release is cut on the 1st of each month by `.github/workflows/monthly-release.yml` when `main` is green and something has merged since the last tag. It is a patch release, or a new minor version if a pull request labelled `breaking` was merged since then (a new major version from 1.0 on), so label pull requests that change behaviour users rely on. - -A patch release is published on its own. A breaking release is created as a draft, without images or a chart: write the upgrade notes in it and publish it, which starts `publish.yml` for the images and the chart. Until then the monthly run releases nothing. - -Scheduled runs only report what they would release until the repository variable `MONTHLY_RELEASE` is set to `true`. The workflow can also be started by hand from the Actions tab, as a dry run by default. - -To release by hand, push a `v*` tag. - ## Questions? Open an issue on GitHub if you have questions or need help getting started. diff --git a/Dockerfile b/Dockerfile index d434e58..7b2795c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM golang:1.27.3-alpine AS builder +FROM golang:1.26.4-alpine AS builder WORKDIR /src @@ -12,17 +12,12 @@ RUN go mod download # Copy source code COPY . . -# Build the binary for the target platform -ARG TARGETARCH -ARG VERSION=dev -ARG COMMIT=unknown -RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ - -ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${COMMIT}" \ - -o /proxy ./cmd/proxy +# Build the binary +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /proxy ./cmd/proxy -FROM alpine:3.24.3 +FROM alpine:3.24.1 -RUN apk add --no-cache ca-certificates git +RUN apk add --no-cache ca-certificates COPY --from=builder /proxy /usr/local/bin/proxy @@ -35,9 +30,5 @@ WORKDIR /data EXPOSE 8080 -# /health checks the database and storage and can take up to ten seconds. -HEALTHCHECK --interval=30s --timeout=11s --start-period=10s --retries=3 \ - CMD wget -q -O /dev/null http://127.0.0.1:8080/health || exit 1 - ENTRYPOINT ["proxy"] CMD ["serve"] diff --git a/README.md b/README.md index dff6964..4609b66 100644 --- a/README.md +++ b/README.md @@ -20,28 +20,6 @@ A 3-day cooldown means that when `lodash` publishes version `4.18.0`, your build Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default and carve out exceptions for packages where you need faster updates. See [docs/configuration.md](docs/configuration.md) for the full config reference. -One instance can offer several cooldown policies through [named profiles](docs/configuration.md#cooldown-profiles). Configure `cooldown.profiles` and point clients at paths such as `/cooldown/3d/npm/` or `/cooldown/7d/cargo/`. Profiles share the package cache; existing URLs keep using the global policy. - -## Artifact Scanning - -Cooldown only looks at a version's publish timestamp — it never inspects the actual bytes. Artifact scanning closes that gap: when enabled, every artifact is staged into storage and scanned by one or more external services (trivy, ClamAV, Wiz, or anything else that speaks a small HTTP/JSON contract) before it's committed to the cache and served to clients. - -```yaml -scanning: - enabled: true - signing_key: ${PROXY_SCANNING_SIGNING_KEY} - scanners: - - name: clamav - url: http://clamav-adapter:8080/scan - mode: block # a block verdict deletes the artifact and returns 403 - - name: trivy - url: http://trivy-adapter:8081/scan - mode: monitor # findings are logged, never gate caching - ecosystems: [npm, pypi] -``` - -The proxy never uploads artifact bytes to a scanner. Each scanner is notified with package metadata plus a short-lived signed URL; the scanner pulls the bytes itself from the proxy's own storage. Scanners run concurrently, and the first `block`-mode scanner to report a verdict of not-allowed wins immediately, canceling the rest. See [docs/configuration.md](docs/configuration.md) for the full config reference and the scanner HTTP contract. - ## Supported Registries | Registry | Language/Platform | Cooldown | Completed | @@ -49,11 +27,11 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | npm | JavaScript | Yes | ✓ | | Cargo | Rust | Yes | ✓ | | RubyGems | Ruby | Yes | ✓ | -| Go proxy | Go | Yes† | ✓ | +| Go proxy | Go | | ✓ | | Hex | Elixir | Yes* | ✓ | | pub.dev | Dart | Yes | ✓ | | PyPI | Python | Yes | ✓ | -| Maven | Java | Yes‡ | ✓ | +| Maven | Java | | ✓ | | Gradle Build Cache | Java/Kotlin | | ✓ | | NuGet | .NET | Yes | ✓ | | Composer | PHP | Yes | ✓ | @@ -61,27 +39,20 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | Conda | Python/R | Yes | ✓ | | CRAN | R | | ✓ | | Julia | Julia | | ✓ | -| Swift | Swift | | ✓ | | Container | Docker/OCI | | ✓ | -| Homebrew | macOS/Linux | | ✓ | | Debian | Debian/Ubuntu | | ✓ | | RPM | RHEL/Fedora | | ✓ | -| Alpine | Alpine Linux | | ✓ | -| Arch | Arch Linux | | ✓ | -| FreeBSD | FreeBSD (ports and pkgbase) | | ✓ | -| Chef Supermarket | Chef | | ✓ | -| Generic | Any | | ✓ | -| Git | Any | | ✓ | -| Helm | Kubernetes | Yes | ✓ | +| Alpine | Alpine Linux | | ✗ | +| Arch | Arch Linux | | ✗ | +| Chef | Chef | | ✗ | +| Generic | Any | | ✗ | +| Helm | Kubernetes | | ✗ | +| Swift | Swift | | ✗ | | Vagrant | Vagrant | | ✗ | Cooldown requires publish timestamps in metadata. Registries without a "Yes" in the cooldown column either don't expose timestamps or haven't been wired up yet. -\* Hex cooldown requires disabling registry signature verification (`HEX_UNSAFE_REGISTRY=1`) since the proxy re-encodes the protobuf payload. Origin verification can stay enabled. - -† Go cooldown uses the time in each version's `.info`, which is the commit time of the tagged revision, not when the tag was pushed. - -‡ Maven cooldown uses the `Last-Modified` header of each version's `.pom`. On Maven Central that is the deploy time; a Nexus or Artifactory upstream may report when it cached the file instead. SNAPSHOT versions are not held back. +\* Hex cooldown requires disabling registry signature verification (`HEX_NO_VERIFY_REPO_ORIGIN=1`) since the proxy re-encodes the protobuf payload. ## Install @@ -89,53 +60,7 @@ Cooldown requires publish timestamps in metadata. Registries without a "Yes" in brew install git-pkgs/git-pkgs/proxy ``` -Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases), or run the container image: - -```bash -docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest -``` - -### Helm - -Install the chart from GHCR, setting the public URL that package-manager clients -will use to reach the proxy: - -```bash -helm install proxy oci://ghcr.io/git-pkgs/charts/proxy \ - --set config.data.base_url=https://proxy.example.com -``` - -The default chart deploys one replica backed by a 10 GiB persistent volume, -using SQLite and filesystem artifact storage under `/data`. See -[`deploy/charts/proxy/values.yaml`](deploy/charts/proxy/values.yaml) for ingress, -external database and object-storage configuration options. - -The SQLite database needs a local disk or a block-backed volume, even with one -replica; `ReadWriteOnce` alone doesn't guarantee that, since the storage class -can still be NFS. If the cluster only offers network filesystem storage, use -Postgres. Artifact storage can stay on NFS, separately from the database. See -[SQLite](docs/configuration.md#sqlite). - -To trust a private CA, for example when upstreams sit behind a TLS-intercepting -corporate proxy or are internal registries signed by an in-house CA, mount the -certificate with `extraVolumes` and `extraVolumeMounts` and add its directory -to `SSL_CERT_DIR`. The proxy process makes all upstream TLS connections itself, -Git fetches included, so no other configuration is needed. Public CAs stay -trusted alongside the private one: - -```yaml -extraVolumes: - - name: corp-ca - configMap: - name: corp-ca # e.g. key corp-ca.crt, PEM-encoded -extraVolumeMounts: - - name: corp-ca - mountPath: /etc/proxy/ca - readOnly: true -extraEnv: - - name: SSL_CERT_DIR - value: /etc/ssl/certs:/etc/proxy/ca -``` +Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases). ## Quick Start @@ -150,8 +75,7 @@ go build -o proxy ./cmd/proxy ./proxy -listen :3000 -base-url https://proxy.example.com ``` -The proxy is now running. Configure your package managers to use it. For image -builds, see [Using proxy in Docker builds](docs/docker-builds.md). +The proxy is now running. Configure your package managers to use it. ## OpenAPI (Swagger) @@ -194,18 +118,6 @@ Or use environment variable: npm_config_registry=http://localhost:8080/npm/ npm install ``` -`npm audit`, `pnpm audit`, `yarn npm audit` and `npm audit signatures` work -through the proxy: the audit and signing-key endpoints are passed through to the -configured upstream registry, with upstream authentication applied. Advisories -therefore come from upstream's database, not from the proxy's own vulnerability -data, and versions withheld by [cooldown](#version-cooldown) are not excluded -from the report. Packuments point each version's attestations (provenance) at -the proxy, which fetches them from the package's registry and caches them; -attestations of versions held back by cooldown or blocked by the denylist are -not served. - -To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds). - ### Cargo Create or edit `~/.cargo/config.toml`: @@ -245,53 +157,12 @@ export GOPROXY=http://localhost:8080/go,direct Or in your shell profile for persistence. -Checksum lookups go through the same URL: the proxy serves `sum.golang.org` at -`/go/sumdb/sum.golang.org/`, and the `go` command uses it without further -setup, so clients need no route of their own to `sum.golang.org`. If the -proxy loses `https://sum.golang.org`, it keeps answering until its cached -copy of the database's signed tree is older than `metadata_ttl`; after that -the `go` command contacts the database directly as before. Set -`upstream.go_sumdbs_offline` on networks where clients cannot do that. Other -checksum databases are listed under `upstream.go_sumdbs`, see -[Go checksum database](docs/configuration.md#go-checksum-database). - -govulncheck can use the proxy for the Go vulnerability database: - -```bash -govulncheck -db http://localhost:8080/go/vulndb ./... -``` - -See [Go vulnerability database](docs/configuration.md#go-vulnerability-database). - -### Homebrew - -Point Homebrew's JSON API and artifact domain at the proxy: - -```bash -export HOMEBREW_API_DOMAIN=http://localhost:8080/homebrew -export HOMEBREW_ARTIFACT_DOMAIN=http://localhost:8080 -``` - -The artifact domain proxies manifests and bottle blobs under `/v2/homebrew/core/`. GHCR routing is limited to that repository. Source archives, cask application downloads, custom tap artifacts, and legacy flat-file bottle mirrors use Homebrew's normal fallback URLs. Keep fallback enabled by leaving `HOMEBREW_ARTIFACT_DOMAIN_NO_FALLBACK` unset. - -Enable `cache_metadata` or set `PROXY_CACHE_METADATA=true` to retain Homebrew JSON API responses for offline fallback. Bottle blobs and their OCI manifests are cached without this setting. `proxy mirror pkg:brew/` pre-fetches a formula's bottles (see [Homebrew bottles](docs/configuration.md#homebrew-bottles)). - -The upstreams default to `https://formulae.brew.sh/api` for the JSON API and `https://ghcr.io` for artifacts. To chain this proxy to another proxy, configure its Homebrew endpoints as the upstreams: - -```yaml -upstream: - homebrew_api: "https://upstream-proxy.example.com/homebrew" - homebrew_artifact: "https://upstream-proxy.example.com" -``` - -The equivalent environment variables are `PROXY_UPSTREAM_HOMEBREW_API` and `PROXY_UPSTREAM_HOMEBREW_ARTIFACT`. - ### Hex (Elixir) -Configure the mirror: +Configure in `~/.hex/hex.config`: -```bash -mix hex.config mirror_url http://localhost:8080/hex +```erlang +{default_url, <<"http://localhost:8080/hex">>}. ``` Or set the environment variable: @@ -300,14 +171,6 @@ Or set the environment variable: export HEX_MIRROR=http://localhost:8080/hex ``` -When the proxy applies a cooldown, also set: - -```bash -export HEX_UNSAFE_REGISTRY=1 -``` - -Filtering releases removes the registry signature. The proxy preserves the package name and repository fields, so origin verification can stay enabled. Without a cooldown, leave signature verification enabled too. - ### pub.dev (Dart/Flutter) Set the PUB_HOSTED_URL environment variable: @@ -349,8 +212,6 @@ Add to your `~/.m2/settings.xml`: The `/maven/` endpoint uses Maven Central as primary upstream and falls back to the Gradle Plugin Portal for Gradle plugin marker metadata and related artifacts when the primary upstream returns not found. -Further Maven repositories can be configured under `upstream.maven_repositories` and are served at `/maven/{name}/`, each from its own upstream (see [Maven repositories](docs/configuration.md#maven-repositories)). Point a second `` or Gradle `maven(url = ...)` entry at `http://localhost:8080/maven/{name}/`. - For Gradle plugin resolution via the same proxy endpoint: ```kotlin @@ -363,20 +224,20 @@ pluginManagement { ### Gradle HTTP Build Cache -Configure in `settings.gradle.kts`: +Configure in `settings.gradle(.kts)`: ```kotlin buildCache { + local { + enabled = false + } remote { url = uri("http://localhost:8080/gradle/") + push = true } } ``` -Enable build caching with `org.gradle.caching=true` in `gradle.properties`, or pass `--build-cache` when running Gradle. The local cache stays enabled and remote uploads are disabled by default. - -To upload from CI, set `gradle.build_cache.read_only` to `false` on the proxy and add `isPush = System.getenv("CI") != null` inside the client's `remote` block. See [docs/configuration.md](docs/configuration.md#gradle-build-cache). - ### NuGet Configure in `nuget.config`: @@ -417,17 +278,6 @@ Or set globally: composer config -g repositories.proxy composer http://localhost:8080/composer ``` -`composer audit`, `audit.block-insecure` and Packagist's malware filter list -work through the proxy when `upstream.composer_repository` offers them in its -`packages.json`, as repo.packagist.org does. The proxy advertises them in its -own `packages.json` and passes the advisories and filter list requests through -to the upstream uncached. Names of packages in `upstream.composer_routes` or -`hosted.composer_vendors` are removed from advisories requests before they are -sent. Versions withheld by [cooldown](#version-cooldown) are not excluded from -the report. - -To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds). - ### Conan (C/C++) Add the proxy as a remote: @@ -439,24 +289,6 @@ conan remote disable conancenter Or configure in `~/.conan2/remotes.json`. -### Chef Supermarket (Berkshelf, Policyfiles) - -Point Berkshelf at the proxy in your `Berksfile`: - -```ruby -source "http://localhost:8080/chef" -``` - -Or set the default source of a Policyfile: - -```ruby -default_source :supermarket, "http://localhost:8080/chef" -``` - -The proxy serves the `/universe` dependency index and the cookbook API, rewriting every download location to itself while leaving dependency constraints unchanged. Cookbook archives are fetched through the Supermarket download redirect and cached, so they stay available when the Supermarket is down. With `cache_metadata` on, the universe and cookbook metadata follow the usual TTL and stale fallback. - -To proxy a private Supermarket instead of `supermarket.chef.io`, set `upstream.chef` (or `PROXY_UPSTREAM_CHEF`) and add credentials under `upstream.auth` if it needs them. See [docs/configuration.md](docs/configuration.md#upstream-registries). - ### Conda Configure in `~/.condarc`: @@ -469,17 +301,12 @@ default_channels: - http://localhost:8080/conda/main ``` -`default_channels` also routes requests for `defaults` through the proxy, including `-c defaults` and `environment.yml` files that name it. - Or set via command: ```bash conda config --add channels http://localhost:8080/conda/main -conda config --add default_channels http://localhost:8080/conda/main ``` -pixi and other rattler-based clients use sharded repodata ([CEP 16](https://github.com/conda/ceps/blob/main/cep-0016.md)) where a channel offers it, as conda-forge does. Point them at the same channel URLs, for example `pixi init -c http://localhost:8080/conda/conda-forge`. The shard index and shards are cached as metadata when `cache_metadata` is on. - ### CRAN (R) Set the repository in R: @@ -498,8 +325,6 @@ local({ }) ``` -Source packages, Windows and macOS binaries (including the `bin/macosx/big-sur-arm64/` layout used by R 4.3 and later) are cached as artifacts. `PACKAGES` indexes and the `src/contrib/Meta/*.rds` files that `remotes::install_version` and pak read are cached as metadata. - ### Julia Set the Pkg server before starting Julia: @@ -515,25 +340,6 @@ ENV["JULIA_PKG_SERVER"] = "http://localhost:8080/julia" using Pkg; Pkg.update() ``` -### Swift - -Configure the proxy as the default registry for the current Swift package: - -```bash -swift package-registry set --allow-insecure-http http://localhost:8080/swift -``` - -Registry dependencies use their scoped package identifier in `Package.swift`: - -```swift -dependencies: [ - .package(id: "apple.swift-argument-parser", from: "1.2.0") -] -``` - -The proxy supports dependency resolution and source downloads. Publishing with -`swift package-registry publish` is not supported. - ### Docker / Container Registry Configure Docker to use the proxy as a registry mirror in `/etc/docker/daemon.json`: @@ -556,112 +362,6 @@ Or pull images directly: docker pull localhost:8080/library/nginx:latest ``` -The registry is pull-only. BuildKit cache exports (`--cache-to type=registry`) -and other pushes need a separate registry. - -SBOMs, signatures and attestations attached to an image are found through the -OCI 1.1 referrers API (`GET /v2/{name}/referrers/{digest}`). `oras discover` -and `notation verify` use it, and so does cosign for its newer signature -bundles. The proxy forwards these requests and caches the returned index with -the same TTL as other metadata, so they keep working offline. It does not -filter by `artifactType`; it returns the full list and the client filters it, -as the spec allows. Registries without the referrers API (GHCR, for example) -answer 404, and clients then look up the `sha256-` tag instead, which -the proxy serves like any other manifest. The proxy is pull-only, so attaching -new artifacts through it is not supported. - -#### containerd (Kubernetes, k3s, nerdctl) - -containerd mirrors send the original registry host in an `ns` query -parameter, so one mirror entry can serve Docker Hub and the registries -configured in `upstream.oci` whose URL has no path. Point containerd's CRI -plugin at a hosts directory in `/etc/containerd/config.toml` and restart -containerd. For containerd 2.x: - -```toml -version = 3 - -[plugins."io.containerd.cri.v1.images".registry] - config_path = "/etc/containerd/certs.d" -``` - -For containerd 1.x: - -```toml -version = 2 - -[plugins."io.containerd.grpc.v1.cri".registry] - config_path = "/etc/containerd/certs.d" -``` - -Then create `/etc/containerd/certs.d/_default/hosts.toml`: - -```toml -[host."http://proxy.example.com:8080"] - capabilities = ["pull", "resolve"] -``` - -To check that CRI picked up the directory, confirm the setting containerd -runs with and pull through CRI (on k3s use `k3s crictl`): - -```bash -containerd config dump | grep config_path -crictl pull docker.io/library/nginx:latest -``` - -The proxy logs a `container manifest request` line for the pull and, when -`access_log.path` is set, an access-log entry. `ctr images pull --hosts-dir` -is no substitute for this check: it reads the directory itself and succeeds -even while CRI still has no `config_path`. - -`docker.io` and the host of `upstream.oci_default` use the default registry; -the host of each `upstream.oci` URL uses that named registry, and so do the -hosts listed for it in `upstream.oci_mirrors`. Pulls for any other registry get -`404 NAME_UNKNOWN`, and containerd falls back to the registry itself. Existing -per-registry `hosts.toml` files that point at `/v2/upstream/{name}` with -`override_path = true` keep working for that upstream's own host and for -Docker Hub. If the upstream is a mirror of the registry the nodes pull from, -say an Artifactory remote of `ghcr.io`, list that registry in -`upstream.oci_mirrors` (`ghcr: ["ghcr.io"]`); otherwise those pulls get the -404 as well. See [docs/configuration.md](docs/configuration.md) for how -hosts are matched and which entry wins when two share a host. k3s generates -the hosts directory itself from `/etc/rancher/k3s/registries.yaml`; `mirrors: -{"*": {endpoint: ["http://proxy.example.com:8080"]}}` produces the same -`_default` entry. - -### Helm - -Configure each HTTP chart repository with a name, then add the matching proxy -URL to Helm: - -```yaml -upstream: - helm: - bitnami: "https://charts.bitnami.com/bitnami" -``` - -```bash -helm repo add bitnami http://localhost:8080/helm/bitnami -helm repo update -helm pull bitnami/nginx -``` - -The proxy caches `index.yaml` using the normal metadata-cache settings and -caches chart archives after verifying their SHA-256 digest from the index. - -For charts stored in an OCI registry, configure a named OCI upstream and add -the reserved `upstream/{name}` prefix to the chart reference: - -```yaml -upstream: - oci: - ghcr: "https://ghcr.io" -``` - -```bash -helm pull oci://localhost:8080/upstream/ghcr/owner/charts/mychart --version 1.0.0 --plain-http -``` - ### Debian / APT Configure APT to use the proxy in `/etc/apt/sources.list.d/proxy.list`: @@ -670,77 +370,24 @@ Configure APT to use the proxy in `/etc/apt/sources.list.d/proxy.list`: deb http://localhost:8080/debian stable main contrib ``` -Or in deb822 format, in `/etc/apt/sources.list.d/proxy.sources`: - -``` -Types: deb -URIs: http://localhost:8080/debian -Suites: stable -Components: main contrib -Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg -``` - -Replace your existing sources.list entries (on newer releases also -`debian.sources` or `ubuntu.sources` in `/etc/apt/sources.list.d/`), then: +Replace your existing sources.list entries, then: ```bash sudo apt update ``` -There is no proxy key to install. The proxy serves the upstream's signed -`InRelease` and `Release` files unchanged, so APT verifies them with the -upstream archive's own key from the distribution's keyring package: -`debian-archive-keyring` on Debian, `ubuntu-keyring` on Ubuntu -(`/usr/share/keyrings/ubuntu-archive-keyring.gpg`). `Signed-By:` in a -`.sources` file, or `[signed-by=...]` in a `.list` line, restricts the source -to that key. For a third-party upstream, use the key its vendor publishes. - -The upstream defaults to `http://deb.debian.org/debian`. To proxy a different APT repository (e.g. Ubuntu), set `upstream.debian` in the config file or `PROXY_UPSTREAM_DEBIAN` in the environment: - -```yaml -upstream: - debian: "http://archive.ubuntu.com/ubuntu" -``` - -A release's security updates are served by a separate archive, so additional -archives are configured under `upstream.debian_repositories` and served at -`/debian/{name}/`: - -```yaml -upstream: - debian: "http://deb.debian.org/debian" - debian_repositories: - security: "https://security.debian.org/debian-security" -``` - -``` -deb http://localhost:8080/debian trixie main -deb http://localhost:8080/debian/security trixie-security main -``` - -`upstream.debian` keeps serving `/debian/pool/…` and `/debian/dists/…` -unchanged. See [Debian archives](docs/configuration.md#debian-archives). - -The version denylist blocks recognized `.deb` downloads but leaves signed APT -indexes unchanged. To keep APT from selecting a denied version, configure -[client-side version pinning](docs/configuration.md#apt-version-pinning). - ### RPM / Yum / DNF -In the `[fedora]` section of your existing repository file under `/etc/yum.repos.d/`, replace the `metalink` or `mirrorlist` entry with: +Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`: ```ini +[proxy-fedora] +name=Fedora via Proxy baseurl=http://localhost:8080/rpm/releases/$releasever/Everything/$basearch/os/ +enabled=1 +gpgcheck=0 ``` -In the `[updates]` section of `fedora-updates.repo`, replace its `metalink` or `mirrorlist` entry with: - -```ini -baseurl=http://localhost:8080/rpm/updates/$releasever/Everything/$basearch/ -``` - -Keep the existing `gpgkey` and signature settings, including `gpgcheck=1`. The proxy serves RPM packages unchanged, so their signatures remain valid. - Then: ```bash @@ -748,262 +395,6 @@ sudo dnf clean all sudo dnf update ``` -### Alpine / apk - -Point `/etc/apk/repositories` at the proxy. The default repository name -`alpine` proxies the official mirror (`https://dl-cdn.alpinelinux.org/alpine`): - -``` -http://localhost:8080/apk/alpine/v3.22/main -http://localhost:8080/apk/alpine/v3.22/community -``` - -Then: - -```bash -apk update -``` - -Repository indexes (v2 `APKINDEX.tar.gz` and v3 `Packages.adb`), detached -signatures, and packages are served byte-for-byte unchanged, so apk's normal -signature verification keeps working. Indexes use the metadata cache -(`metadata_ttl`, stale fallback); `.apk` packages are stored in the shared -artifact cache and remain available when the upstream is unreachable. - -Set `upstream.apk_snapshots` (e.g. `"90d"`) to keep replaced indexes and serve -them by date at `/apk/alpine@2026-10-01/v3.22/main`, so pinned package versions -stay installable after the mirror moves on. See -[Alpine APK snapshots](docs/configuration.md#alpine-apk-snapshots). - -To proxy other mirrors or private repositories, configure named upstreams -under `upstream.apk` (this replaces the built-in default; re-add `alpine` if -you still want it): - -```yaml -upstream: - apk: - alpine: "https://dl-cdn.alpinelinux.org/alpine" - private: "https://apk.example.com" -``` - -``` -http://localhost:8080/apk/private -``` - -apk appends the architecture and index filename to each repository line -itself. - -### Arch Linux / pacman - -Put the proxy at the top of `/etc/pacman.d/mirrorlist`. The default repository -name `archlinux` proxies the official GeoIP mirror -(`https://geo.mirror.pkgbuild.com`): - -``` -Server = http://localhost:8080/arch/archlinux/$repo/os/$arch -``` - -Then: - -```bash -pacman -Syu -``` - -Repository databases (`core.db`, `core.files` and their `.tar.*` forms), -database signatures, packages and package signatures are served byte-for-byte -unchanged, so pacman's signature checks keep working. Databases use the -metadata cache (`metadata_ttl`, stale fallback). Each database refresh also -fetches its signature and stores it for exactly that database, so a signature -is only ever served with the database it signs. Enable `cache_metadata` (or set `PROXY_CACHE_METADATA=true`) so -`pacman -Sy` keeps working while the upstream is down; without it databases are -fetched fresh on every request. Packages and their `.sig` files are stored in the shared artifact -cache and remain available when the upstream is unreachable. - -The proxy only starts sending a package once it has the whole file, and pacman -gives up on a download that sends nothing for 10 seconds. The fetch keeps -running and the next try is served from the cache. On a slow upstream, set -`DisableDownloadTimeout` in `/etc/pacman.conf` to avoid the failed first try. - -`extra.files` is about 50 MB and is read into memory on each `pacman -F` -request. Raise `metadata_max_size` (100 MB by default) for larger custom -repositories. - -A custom repository that rebuilds a package under the same version leaves the -old package and signature in the cache. Bump `pkgrel` instead, or remove the -cached version. - -#### Arch Linux ARM - -Arch Linux ARM has no built-in repository. Add one under `upstream.arch`, using a -regional mirror from that serves HTTPS: - -```yaml -upstream: - arch: - archlinux: "https://geo.mirror.pkgbuild.com" - archlinuxarm: "https://.mirror.archlinuxarm.org" -``` - -Setting `upstream.arch` replaces the built-in default, so keep `archlinux` in the map -if you also proxy x86_64 Arch. - -Arch Linux ARM mirrors use `$arch/$repo` instead of `$repo/os/$arch`. In -`/etc/pacman.d/mirrorlist`: - -``` -Server = http://localhost:8080/arch/archlinuxarm/$arch/$repo -``` - -Avoid `mirror.archlinuxarm.org` as the upstream. It redirects each request to a -changing regional mirror over plain HTTP, so the database and the packages can come -from mirrors with a different sync state. - -### FreeBSD / pkg - -Override the repositories in `/usr/local/etc/pkg/repos/FreeBSD.conf`. Settings -there are merged with `/etc/pkg/FreeBSD.conf`, so only the URL and mirror type -change and the signature settings stay. The default repository name `freebsd` -proxies the official mirror (`https://pkg.freebsd.org`). On FreeBSD 15: - -``` -FreeBSD-ports: { - url: "http://localhost:8080/freebsd/freebsd/${ABI}/quarterly", - mirror_type: "none" -} -FreeBSD-ports-kmods: { - url: "http://localhost:8080/freebsd/freebsd/${ABI}/kmods_quarterly_${VERSION_MINOR}", - mirror_type: "none" -} -FreeBSD-base: { - url: "http://localhost:8080/freebsd/freebsd/${ABI}/base_release_${VERSION_MINOR}", - mirror_type: "none" -} -``` - -FreeBSD 14 has a single repository called `FreeBSD`: - -``` -FreeBSD: { - url: "http://localhost:8080/freebsd/freebsd/${ABI}/quarterly", - mirror_type: "none" -} -``` - -Then: - -```bash -pkg update -``` - -Keep the path after `${ABI}` that `pkg -vv` shows for each repository -(`quarterly`, `latest`, `base_latest`, ...) and replace only the host part. - -The repository catalogues (`meta.conf`, `packagesite.pkg`, `data.pkg`) carry -their signature inside the archive and are served byte-for-byte unchanged -through the metadata cache (`metadata_ttl`, stale fallback). Enable -`cache_metadata` so `pkg update` keeps working while the upstream is down. -Packages under `Hashed/` have their content hash in the file name, so they are -stored in the shared artifact cache as `pkg:bsd/freebsd/{name}@{version}` and -remain available when the upstream is unreachable. - -Other files are passed through uncached. That includes the bootstrap package -under `Latest/`, whose signature file is rewritten with every repository build, -and package files without a hash in their name, which can be rebuilt under the -same name. The denylist still applies to unhashed package files. Since only -cached artifacts can be scanned, both kinds of package file are refused while -a scanner covers `freebsd`; use repositories built with `pkg repo --hash` and -bootstrap pkg without the proxy. - -### GitHub Releases / mise (aqua backend) - -Configure named generic upstreams: - -```yaml -upstream: - generic: - github: "https://github.com" - github-api: "https://api.github.com" -``` - -Then rewrite GitHub URLs in mise's settings (`~/.config/mise/config.toml`, mise ≥ 2025.9.3): - -```toml -[settings.url_replacements] -"regex:^https://github\\.com/([^/]+)/([^/]+)/releases/download/(.+)" = "http://localhost:8080/generic/github/$1/$2/releases/download/$3" -"regex:^https://api\\.github\\.com/(.*)" = "http://localhost:8080/generic/github-api/$1" -``` - -Release assets are cached permanently after the first download and keep -installing while GitHub is down. Tag lookups through `api.github.com` are -cached for `metadata_ttl` and served stale during an outage or rate limit. -Commit a `mise.lock` and install with `mise install --locked` so pinned -installs need no API call at all. - -If the fleet exceeds GitHub's anonymous rate limit, add a bearer token for -`https://api.github.com` under `upstream.auth`. Everyone who can reach the -proxy can then make requests with that token, so the proxy requires -`generic_paths` listing the API paths it may serve, and the token should be a -fine-grained one with read-only access to public repositories: - -```yaml -upstream: - generic_paths: - github-api: - - "repos/*/*/releases" - - "repos/*/*/releases/**" - - "repos/*/*/tags" - auth: - "https://api.github.com": - type: bearer - token: "${GITHUB_TOKEN}" -``` - -Maven, mvnd, Node and Go downloads are cached the same way once their hosts -are configured as generic upstreams; their version-pinned paths are built in -(see [configuration](docs/configuration.md)): - -```yaml -upstream: - generic: - apache: "https://archive.apache.org" - nodejs: "https://nodejs.org" - golang: "https://dl.google.com/go" -``` - -```toml -[settings] -node.mirror_url = "http://localhost:8080/generic/nodejs/dist/" -go.download_mirror = "http://localhost:8080/generic/golang" - -[settings.url_replacements] -"https://archive.apache.org/" = "http://localhost:8080/generic/apache/" -``` - -Other download hosts need their version-pinned paths declared in -`upstream.generic_artifacts`. - -### Git repositories - -Configure named Git servers: - -```yaml -upstream: - git: - github: "https://github.com" -``` - -Then point clones at the proxy: - -```bash -git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/ -``` - -Each repository is cached as a bare clone and refreshed whenever its branches -or tags change upstream. Git LFS objects are cached in the artifact cache; -git-lfs uses the proxy for them with no further setup. See [configuration](docs/configuration.md#git-repositories) -for setting the rewrite through environment variables, size limits, private -repositories and outage behavior. - ## Configuration The proxy can be configured via: @@ -1018,14 +409,13 @@ The proxy can be configured via: -config string Path to configuration file -listen string Address to listen on (default ":8080") -base-url string Public URL of this proxy (default "http://localhost:8080") --storage-url string Storage URL (file://, s3://, gs://, azblob://) +-storage-url string Storage URL (file:// or s3://) -storage-path string Path to artifact storage directory (deprecated, use -storage-url) -database-driver string Database driver: sqlite or postgres (default "sqlite") -database-path string Path to SQLite database file (default "./cache/proxy.db") -database-url string PostgreSQL connection URL -log-level string Log level: debug, info, warn, error (default "info") -log-format string Log format: text, json (default "text") --access-log string Path to the JSONL access log -version Print version and exit ``` @@ -1041,8 +431,6 @@ PROXY_DATABASE_PATH=./cache/proxy.db PROXY_DATABASE_URL=postgres://user:pass@localhost/proxy?sslmode=disable PROXY_LOG_LEVEL=info PROXY_LOG_FORMAT=text -PROXY_ACCESS_LOG_PATH=/var/log/proxy/access.jsonl -PROXY_UPSTREAM_SWIFT=https://tuist.dev/api/registry/swift ``` ### Configuration File @@ -1054,8 +442,6 @@ base_url: "http://localhost:8080" storage: url: "file:///var/cache/proxy/artifacts" max_size: "10GB" # Optional: evict LRU when exceeded - retention: - default: "30d" # Optional: evict unused artifacts after 30 days; excludes generic database: driver: "sqlite" @@ -1065,22 +451,16 @@ log: level: "info" format: "text" -access_log: - path: "/var/log/proxy/access.jsonl" # Optional JSONL activity log - # Optional: override upstream URLs upstream: npm: "https://registry.npmjs.org" cargo: "https://index.crates.io" - swift: "https://tuist.dev/api/registry/swift" # Optional: version cooldown (see above) cooldown: default: "3d" ``` -See the [configuration reference](docs/configuration.md#upstream-registries) for every upstream key, environment variable, and default URL. - Run with config file: ```bash @@ -1089,7 +469,7 @@ Run with config file: ### PostgreSQL -SQLite is the default and works well for single-node deployments, with the database file on a local disk or block-backed volume. For multi-node setups, when only network filesystem storage such as NFS is available, or if you prefer a managed database, switch to Postgres: +SQLite is the default and works well for single-node deployments. For multi-node setups or if you prefer a managed database, switch to Postgres: ```yaml database: @@ -1124,57 +504,6 @@ storage: Set credentials via standard AWS environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`). -### Google Cloud Storage - -The proxy can store cached artifacts in a GCS bucket using the `gs://` URL scheme. - -```yaml -storage: - url: "gs://my-bucket-name" -``` - -Authentication uses [Application Default Credentials](https://docs.cloud.google.com/docs/authentication/application-default-credentials), which means no credentials need to be embedded in the config or environment. Supported sources, in order: - -- **GKE Workload Identity** — bind the Kubernetes service account running the proxy to a Google service account that has `roles/storage.objectAdmin` on the bucket. The proxy will use the workload's token automatically. -- **Attached service account** on GCE, Cloud Run, Cloud Functions, etc. -- **`GOOGLE_APPLICATION_CREDENTIALS`** environment variable pointing at a service account JSON key file. -- **`gcloud auth application-default login`** for local development. - -#### GKE Workload Identity setup - -```bash -# 1. Create a Google service account -gcloud iam service-accounts create git-pkgs-proxy \ - --project=PROJECT_ID - -# 2. Grant it access to the bucket -gsutil iam ch \ - serviceAccount:git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com:objectAdmin \ - gs://my-bucket-name - -# 3. Bind the Kubernetes service account to it -gcloud iam service-accounts add-iam-policy-binding \ - git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com \ - --role=roles/iam.workloadIdentityUser \ - --member="serviceAccount:PROJECT_ID.svc.id.goog[NAMESPACE/KSA_NAME]" - -# 4. Annotate the Kubernetes service account -kubectl annotate serviceaccount KSA_NAME \ - --namespace=NAMESPACE \ - iam.gke.io/gcp-service-account=git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com -``` - -#### Direct serve (signed URLs) with Workload Identity - -When `direct_serve: true` is enabled, the proxy issues HTTP 302 redirects to presigned GCS URLs. Workload Identity provides no private key, so the GCS backend calls the [IAM Credentials `signBlob` API](https://docs.cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/signBlob). Grant the service account the token-creator role on itself: - -```bash -gcloud iam service-accounts add-iam-policy-binding \ - git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com \ - --role=roles/iam.serviceAccountTokenCreator \ - --member="serviceAccount:git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com" -``` - ## CLI Commands ### serve (default) @@ -1197,9 +526,6 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 # Mirror all versions of a package proxy mirror pkg:npm/lodash -# Mirror only the newest version of a package -proxy mirror --latest pkg:npm/lodash - # Mirror from a CycloneDX or SPDX SBOM proxy mirror --sbom sbom.cdx.json @@ -1208,17 +534,9 @@ proxy mirror --dry-run pkg:npm/lodash # Control parallelism proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21 - -# Mirror a container image for one platform -proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20 - -# Mirror a commit of an upstream.git repository into the git cache -proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 ``` -The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror. - -With `scanning.enabled`, the mirror command refuses every package and container image so nothing is cached unscanned. Each one is reported as failed, so the command exits with status 1. Scanners fetch the staged bytes from the running server, which this command does not start. Use the mirror API below instead, since its jobs run inside the server and go through the pre-cache scan. `--dry-run` still lists what would be mirrored. +The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. A mirror API is also available when the server is running: @@ -1228,11 +546,6 @@ curl -X POST http://localhost:8080/api/mirror \ -H "Content-Type: application/json" \ -d '{"purls": ["pkg:npm/lodash@4.17.21"]}' -# Start a mirror job from an inline CycloneDX or SPDX JSON SBOM -curl -X POST http://localhost:8080/api/mirror \ - -H "Content-Type: application/json" \ - -d '{"sbom":{"bomFormat":"CycloneDX","components":[{"purl":"pkg:npm/lodash@4.17.21"}]}}' - # Check job status curl http://localhost:8080/api/mirror/mirror-1 @@ -1240,18 +553,6 @@ curl http://localhost:8080/api/mirror/mirror-1 curl -X DELETE http://localhost:8080/api/mirror/mirror-1 ``` -### hosted - -Push package archives to a running proxy with `hosted.enabled`, list them and delete them. See [Hosted Packages](docs/configuration.md#hosted-packages). - -```bash -export PROXY_URL=https://proxy.example.com PROXY_HOSTED_TOKEN=... -proxy hosted push npm request 2.88.3 ./request-2.88.3.tgz -proxy hosted push composer monolog/monolog 3.5.0 ./monolog-3.5.0.zip -proxy hosted list npm request -proxy hosted delete npm request 2.88.3 -``` - ### stats Show cache statistics without running the server. @@ -1306,15 +607,10 @@ Recently cached: | Endpoint | Description | |----------|-------------| | `GET /` | Dashboard (web UI) | -| `GET /health` | Health check and upstream circuit breaker state (JSON; HTTP 200 healthy, 503 unhealthy) | +| `GET /health` | Health check (JSON; HTTP 200 healthy, 503 unhealthy) | | `GET /stats` | Cache statistics (JSON) | | `GET /metrics` | Prometheus metrics | | `GET /npm/*` | npm registry protocol | -| `POST /npm/-/npm/v1/security/*` | npm/pnpm/Yarn audit endpoints, passed through to upstream | -| `GET /npm/-/npm/v1/keys` | npm registry signing keys, passed through to upstream | -| `GET /npm/-/package/{name}/dist-tags` | `npm dist-tag ls`, answered from the packument the proxy serves (cooldown and pushed versions applied); changing tags is not supported | -| `GET /npm/-/npm/v1/attestations/{name}@{version}` | npm provenance attestations of a version, fetched from the package's registry and cached | -| `GET /npm/-/v1/search`, `GET /npm/-/ping` | `npm search` and `npm ping`, passed through to upstream uncached | | `GET /cargo/*` | Cargo sparse index protocol | | `GET /gem/*` | RubyGems protocol | | `GET /go/*` | Go module proxy protocol | @@ -1324,64 +620,22 @@ Recently cached: | `GET /maven/*` | Maven repository protocol | | `GET /nuget/*` | NuGet V3 API | | `GET /composer/*` | Composer/Packagist protocol | -| `GET`, `POST /composer/api/security-advisories/` | Composer security advisories, passed through to upstream without private package names | -| `GET /composer/lists/summary.json` | Composer filter list summary, passed through to upstream uncached | | `GET /conan/*` | Conan C/C++ protocol | -| `GET /chef/*` | Chef Supermarket protocol (universe, cookbook API and downloads) | | `GET /conda/*` | Conda/Anaconda protocol | | `GET /cran/*` | CRAN (R) protocol | | `GET /julia/*` | Julia Pkg server protocol | -| `GET /swift/*` | Swift Package Registry v1 protocol | -| `GET /helm/{repository}/*` | HTTP Helm chart repository protocol | -| `GET /homebrew/*` | Homebrew JSON API | | `GET /v2/*` | OCI/Docker registry protocol | -| `GET /v2/homebrew/core/*` | Homebrew core bottle manifests and blobs from GHCR | -| `GET /apk/{repository}/*` | Alpine APK repository protocol | -| `GET /arch/{repository}/*` | Arch Linux pacman repository protocol | -| `GET /freebsd/{repository}/*` | FreeBSD pkg repository protocol | -| `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) | -| `GET /git/{name}/{repo}/info/refs`, `POST /git/{name}/{repo}/git-upload-pack` | Git smart HTTP protocol, read-only | -| `GET /debian/*` | Debian/APT repository protocol (main archive) | -| `GET /debian/{repository}/*` | Debian/APT repository protocol (named archive, e.g. security) | +| `GET /debian/*` | Debian/APT repository protocol | | `GET /rpm/*` | RPM/Yum repository protocol | ### Mirror API | Endpoint | Description | |----------|-------------| -| `POST /api/mirror` | Start a mirror job (JSON body with `purls` or an inline `sbom`) | +| `POST /api/mirror` | Start a mirror job (JSON body with `purls`) | | `GET /api/mirror/{id}` | Get job status and progress | | `DELETE /api/mirror/{id}` | Cancel a running job | -### Hosted API - -Registered when `hosted.enabled` is set. Pushing and deleting require `Authorization: Bearer `. - -| Endpoint | Description | -|----------|-------------| -| `PUT /api/hosted/{ecosystem}/{name}/{version}` | Push the request body as the archive for a version (npm and composer) | -| `GET /api/hosted/{ecosystem}/{name}` | List the archives pushed for a package | -| `DELETE /api/hosted/{ecosystem}/{name}/{version}` | Delete a pushed version | - -### Projects API - -Registered when `projects.enabled` is set. Uploading and deleting require `Authorization: Bearer `. See [Project inventory](docs/configuration.md#project-inventory). - -| Endpoint | Description | -|----------|-------------| -| `PUT /api/projects/{project}/sbom` | Replace a project's recorded package versions with the ones the SBOM in the body lists | -| `GET /api/projects` | List projects, or with `?purl=pkg:npm/lodash@4.17.20` the projects using a package or version | -| `GET /api/projects/{project}` | A project's recorded package versions | -| `DELETE /api/projects/{project}` | Delete a project | - -### Cache API - -Registered when `cache_delete` is set. It has no authentication of its own. - -| Endpoint | Description | -|----------|-------------| -| `DELETE /api/cache/{ecosystem}/{name}/{version}` | Delete a version's cached files, or only `?filename=` | - ### Enrichment API The proxy provides REST endpoints for package metadata enrichment, vulnerability scanning, and outdated detection. @@ -1535,7 +789,9 @@ Response: { "cached_artifacts": 142, "total_size_bytes": 523456789, - "total_size": "499.2 MB" + "total_size": "499.2 MB", + "storage_url": "file:///path/to/cache/artifacts", + "database_path": "./cache/proxy.db" } ``` @@ -1567,7 +823,6 @@ Response: The proxy serves a web UI under `/ui`. No separate frontend build is needed -- templates and assets are embedded in the binary. `GET /` redirects to `/ui/`. The UI is mounted under its own prefix so a reverse proxy can apply different access rules to it than to the package endpoints (for example, requiring auth for `PathPrefix(/ui)` while leaving `/npm`, `/pypi` etc. open to build machines). - **Dashboard** (`/ui/`) -- cache stats, popular packages, recently cached artifacts, and vulnerability overview. -- **Analytics** (`/ui/analytics`) -- accumulated download size as a ring broken down by ecosystem with the total in the middle, the cache size, artifact, package and version counts, a per-ecosystem table, the vulnerability overview, and a Runtime card mirroring every counter `/metrics` exposes. See [Analytics](#analytics). - **Install guide** (`/ui/install`) -- per-ecosystem configuration instructions, so you don't have to look them up here. - **Package browser** (`/ui/packages`) -- browse all cached packages with filtering by ecosystem and sorting by hits, size, name, or vulnerability count. - **Search** (`/ui/search?q=...`) -- search cached packages by name. @@ -1575,7 +830,6 @@ The proxy serves a web UI under `/ui`. No separate frontend build is needed -- t - **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts. - **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews. - **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files. -- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated. Package pages then show which projects use each version. ## Monitoring @@ -1583,123 +837,18 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre | Metric | Type | Labels | Description | |--------|------|--------|-------------| -| `proxy_requests_total` | counter | `ecosystem`, `status` | Proxy responses by package ecosystem and HTTP status | -| `proxy_request_duration_seconds` | histogram | `ecosystem`, `status` | Proxy request duration | -| `proxy_cache_hits_total` | counter | `ecosystem`, `kind` | Cache hits. `kind` is `artifact` or `metadata`. | -| `proxy_cache_misses_total` | counter | `ecosystem`, `kind` | Cache misses. `kind` is `artifact` or `metadata`; metadata misses are counted only while `cache_metadata` is on. | +| `proxy_cache_hits_total` | counter | `ecosystem` | Cache hits | +| `proxy_cache_misses_total` | counter | `ecosystem` | Cache misses | | `proxy_cache_size_bytes` | gauge | | Total size of cached artifacts | | `proxy_cached_artifacts_total` | gauge | | Number of cached artifacts | -| `proxy_metadata_cache_served_bytes_total` | counter | `ecosystem` | Metadata bytes served from the metadata cache. | -| `proxy_metadata_cache_size_bytes` | gauge | `ecosystem` | Size of cached metadata documents per ecosystem. | -| `proxy_metadata_cache_entries` | gauge | `ecosystem` | Number of cached metadata documents per ecosystem. | -| `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) | -| `proxy_metadata_evicted_total` | counter | `ecosystem` | Cached metadata documents evicted by the size limit | -| `proxy_metadata_expired_by_change_feed_total` | counter | `ecosystem` | Cached metadata documents set to revalidate because an upstream [change feed](docs/configuration.md#change-feeds) listed them | -| `proxy_fetch_lease_waits_total` | counter | `kind` | Cache misses (`artifact` or `metadata`) that waited for another instance sharing the database to fetch them | | `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream | | `proxy_upstream_errors_total` | counter | `ecosystem`, `error_type` | Upstream fetch failures | | `proxy_storage_operation_duration_seconds` | histogram | `operation` | Storage read/write latency | | `proxy_storage_errors_total` | counter | `operation` | Storage read/write failures | | `proxy_active_requests` | gauge | | In-flight requests | | `proxy_health_probe_failures_total` | counter | `step` | Storage health probe failures by failing step (`write`, `size`, `read`, `verify`, `delete`). | -| `proxy_circuit_breaker_state` | gauge | `registry` | Artifact-fetch circuit breaker state per upstream registry (0 closed, 2 open). Published once that registry's breaker has tripped. | -| `proxy_circuit_breaker_trips_total` | counter | `registry` | Circuit breaker trips per upstream registry. | -| `proxy_upstream_rate_limit_holds_total` | counter | `host` | Holds placed on an upstream host after it rate limited the proxy with HTTP 429, or 503 with a `Retry-After`. See `upstream_holds` under [Health Check](#health-check). | -| `proxy_ecosystem_downloaded_bytes` | gauge | `ecosystem` | Accumulated bytes served from cache: cache hits multiplied by the artifact size they served. | -| `proxy_ecosystem_artifact_downloads` | gauge | `ecosystem` | Accumulated artifact downloads served from cache. | -| `proxy_ecosystem_cache_size_bytes` | gauge | `ecosystem` | Size of cached artifacts per ecosystem. | -| `proxy_ecosystem_cached_artifacts` | gauge | `ecosystem` | Number of cached artifacts per ecosystem. | -| `proxy_ecosystem_packages` | gauge | `ecosystem` | Known packages per ecosystem. | -| `proxy_ecosystem_versions` | gauge | `ecosystem` | Known package versions per ecosystem. | -| `proxy_response_bytes_total` | counter | `ecosystem` | Response body bytes written to clients. Route-labelled, see the label caveat below. | -| `proxy_client_requests_total` | counter | `client` | Requests by client tool, from the User-Agent. | -| `proxy_client_response_bytes_total` | counter | `client` | Response bytes by client tool. | -The `ecosystem` label on `proxy_requests_total` and `proxy_request_duration_seconds` is the mounted route a request arrived on, not the ecosystem recorded against the package it served: `/gem` reports as `rubygems`, `/go` as `golang`, `/composer` as `packagist`, `/apk` as `alpine` and `/v2` as `oci`. Anything outside a package route -- the UI, `/health`, `/metrics`, `/stats` -- reports as `other`, and so did `/apk`, `/helm`, `/homebrew`, `/generic` and `/swift` before they were listed; traffic on those five routes now appears under its own name instead. - -Cache size, artifact count and the per-ecosystem gauges are refreshed every 60 seconds, from a single pass over the database. Circuit breaker state is read from the fetcher on each scrape of `/metrics` and each `/health` request, so `proxy_circuit_breaker_trips_total` counts the trips visible between those reads — a breaker that opens and recovers entirely between two scrapes is not counted. The remaining metrics update on each request. - -The breaker metrics carry one series per upstream host, but only for hosts whose breaker has tripped at least once since startup. A breaker is created per host the proxy fetches artifacts from, and for some ecosystems that host comes from upstream metadata rather than from configuration (composer takes it from a package's `dist.url`, helm from the chart URLs in `index.yaml`), so publishing every host would let upstream content grow the series count for the lifetime of the process. Once a host has tripped it keeps reporting, so a recovery still shows up as a transition to 0 rather than as a series that vanishes. `/health` is not a persistent time series and lists every breaker, tripped or not. - -The `registry` label is the host of the URL the artifact was fetched from. Because that URL can come from upstream metadata, it is not always one a host can be read off — a signed `dist.url` that fails to parse, for instance — and such a breaker is labelled `hostless-url-` instead, where the digest is keyed by a value drawn fresh at startup. Neither `/metrics` nor `/health` requires authentication, so a fetch URL is never published as a label or a key; the digest identifies the breaker for as long as the process runs without revealing the URL behind it or letting a chosen URL be matched against it. - -Alert on `proxy_circuit_breaker_state == 2` sustained for more than a few minutes: while a breaker is open, artifact downloads for that upstream fail with HTTP 502 on every cache miss, and only a single probe request per backoff interval reaches the upstream. Cached artifacts keep serving, and so does metadata for the same ecosystem (metadata does not go through the circuit breaker), so installs fail in a way that looks like a partial upstream outage. - -#### Accumulated download size - -`proxy_ecosystem_downloaded_bytes` is, for every cached artifact, the number of times it was served multiplied by its size. It answers "how much traffic has this proxy actually carried", which is the number that matters when sizing egress or justifying the cache. Two properties are worth knowing before alerting on it. - -**It counts cache hits, not upstream fetches.** The request that first pulls an artifact through the proxy is a miss and is not counted; only later hits are. So the accumulated total is also the upstream bandwidth the cache has saved, not the total bytes the proxy has ever sent. - -**Eviction removes history.** Evicting an artifact clears its size, so its past hits drop out of the total. That is why these are gauges rather than counters, and why the figure can step downwards. Chart them with `max_over_time` rather than `increase`, and read a drop after an eviction sweep as expected rather than as data loss. - -Nothing at the schema level ties `artifacts.version_purl` to a version row, so a cached artifact can end up with no ecosystem to attribute it to. Those are reported under the ecosystem `unattributed` rather than dropped, which keeps the per-ecosystem figures adding up to `proxy_cache_size_bytes` and `proxy_cached_artifacts_total`. A non-zero `unattributed` means the database holds artifact rows whose version or package rows have gone missing. - -The `ecosystem` label on these six is taken from the package record and normalized, so aliases collapse: a database carrying both `gem` and `rubygems` rows -- the proxy writes the former, git-pkgs the latter -- reports one `rubygems` series with the two summed. - -### Analytics - -`/ui/analytics` reports the accumulated download size as a ring broken down by ecosystem, the cache figures from the dashboard, a per-ecosystem table, the vulnerability overview, and a **Runtime** card covering every remaining metric `/metrics` exposes. - -The ring shows at most six slices, because part-to-whole stops being readable past that. When more ecosystems are active the smallest are folded into a single "Other" slice; the table below lists every one of them, so nothing is hidden, only summarised. - -#### No history is kept - -The proxy stores no time series. The page reads the database and the in-process metric registry at request time and reports current state; there is nowhere for it to read yesterday's figures from, and nothing is written for tomorrow. That splits the figures in two, and the page says which is which. - -**Database-derived figures survive a restart.** Download volume, cache size and the package, version and artifact counts come from the `artifacts`, `packages` and `versions` tables, so they are as durable as the database. - -**Registry-derived figures do not.** Everything in the Runtime card -- request counts and latencies, cache hit rate, upstream and storage errors, circuit breaker state, scan results -- lives only in this process's Prometheus registry and starts from zero on restart. A small number there next to a large one above just means the proxy started recently. - -For history, trends and alerting, scrape `/metrics` with Prometheus. That is the intended split: the UI answers "what is true now", Prometheus answers "what happened". - -The same figures are available as JSON from `GET /stats`, which reports `downloaded_bytes`, `downloads` and an `ecosystems` array carrying the per-ecosystem breakdown, served from the same 60-second snapshot the page and the gauges read. When the aggregation fails with no snapshot to fall back on, the response carries `stats_unavailable: true` rather than passing zeros off as a count -- the endpoint keeps answering with the artifact count and cache size either way. - -#### Artifact and metadata caches - -The proxy keeps two caches. Artifacts (tarballs, wheels, crates and so on) live in storage and are what `proxy_cache_size_bytes`, `proxy_ecosystem_downloaded_bytes` and the other artifact figures describe. Registry metadata is cached separately when `cache_metadata` is on. It is stored persistently in the `metadata_cache` table rather than held in memory. - -The `kind` label on `proxy_cache_hits_total` and `proxy_cache_misses_total` keeps the two apart, so an artifact hit rate is `sum(rate(proxy_cache_hits_total{kind="artifact"}[5m]))` over the same expression with misses added. Metadata has its own size and entry gauges plus `proxy_metadata_cache_served_bytes_total`. That one is a process counter rather than a database total, because metadata hits are not recorded per row. On `/ui/analytics` metadata appears in its own card and in the Runtime card's metadata lookups, never mixed into the artifact figures. The card and the two gauges read one 60-second snapshot, like the artifact figures. The bundled Grafana dashboard's hit ratio covers artifacts only, while its hits and misses panel shows each kind as its own series. - -#### Three ecosystem label sets - -`ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which. - -**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_metadata_expired_by_change_feed_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. - -**From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all. - -**From the handler's own name.** `proxy_upstream_fetch_duration_seconds` and `proxy_upstream_errors_total`, which report `composer`, `gem` and `go` where the other two sets report `packagist`, `rubygems` and `golang`. These are published series and are deliberately left as they are; renaming them would break existing queries and alerts. - -### Request sources - -Package managers do not say who invoked them. A request from `pip` or `go` carries a `Host`, an `Accept` and a `User-Agent` -- no `Referer`, no originating URL, nothing naming a repository, pipeline or job. Whatever identity you want has to come from something on the wire, so the proxy attributes requests by the two things always present. - -**Address** -- the TCP peer, or the leftmost `X-Forwarded-For` entry when `trust_forwarded_for` is enabled. Enable that only behind a load balancer or ingress that sets the header; any client can send it, so in front of one it lets a caller forge its own attribution and, by cycling synthetic addresses, fill the table and push every genuine caller into the overflow row. The totals stay correct; the attribution is what is lost. - -**Client** -- the tool, taken from the leading User-Agent token: `pip`, `npm`, `go`, `docker`, `apt`, `curl` and so on. Anything unrecognised reports as `other`. - -Set `ui_request_sources: true` and both appear on `/ui/analytics` under **Runtime -> Request sources**, as a table of the busiest callers by bytes downloaded plus a per-tool breakdown. The table is in-memory and process-lifetime, like the rest of that card. It tracks 200 callers, evicting the least recently seen once full, and summarises everything it is not showing individually -- both evicted callers and those ranked below the display limit -- in a single "other callers" row, so the rows always add up to the totals above them. - -**The flag defaults to off because the page is not authenticated.** `/ui` carries no auth of its own -- it is mounted under its own prefix so a reverse proxy *can* gate it separately, as [Behind a Reverse Proxy](#behind-a-reverse-proxy) describes, but nothing makes you -- and until now it exposed only package data. The sources table changes what is on offer: anyone who can reach the proxy can read the addresses of your build fleet, which tool each runs, and how much each pulled. Turn it on once `/ui` is gated, or leave it off and read the same detail from the access log. - -**What this can and cannot tell you.** How much an address gives you depends entirely on your network. A fleet of build machines with stable addresses attributes cleanly. Containerised CI usually does not: with Docker or Kubernetes executors every job gets an ephemeral address, and egress is commonly NAT'd behind one gateway, so you get runner-node or gateway granularity, not per-project. If you need per-project attribution the caller has to send something naming itself -- a basic-auth username, or a per-project base URL -- which the proxy does not currently read. Say so and it can be added. - -**Why addresses are not Prometheus labels.** Client tool names are exported as `proxy_client_requests_total{client}` because they come from a closed set. Addresses are not exported at all: the caller set is unbounded and outside the proxy's control, and every new address would create a time series that lives forever in your TSDB. The same goes for anything job-scoped -- a pipeline ID must never become a label. Per-address and per-request detail belongs in the access log, which records `remote_ip`, `user_agent`, `client`, `ecosystem` and `bytes` on every line as JSONL, ready for `jq`, Loki or whatever you ship logs to. - -### Failure details - -The failure counts under **Runtime** on `/ui/analytics` -- upstream errors, storage errors, integrity failures, health probe failures and scan errors -- say how often something failed, not what. Set `ui_failure_details: true` and each count gets a **Latest** list underneath it, newest first: the package URL and filename, the upstream URL or storage path, the error, and how long ago it happened. - -The list keeps the last 20 failures per row, in memory only, so it starts empty when the proxy starts, like the counts above it. Credentials, query strings and fragments are stripped from every URL before it is kept, including URLs inside error messages, since that is where registry tokens and signed-URL signatures live. - -The flag defaults to off for the same reason as `ui_request_sources`: package names and upstream hosts say what your users depend on, and `/ui` is not authenticated. Package names and URLs are never Prometheus labels; the same detail is in the proxy's log. - -### Grafana dashboard - -A ready-made dashboard lives at [`deploy/grafana/git-pkgs-proxy.json`](deploy/grafana/git-pkgs-proxy.json). Import it via **Dashboards -> New -> Import** and pick your Prometheus data source when prompted; it has no hardcoded data source UID. - -It carries three ecosystem filters rather than one, because `ecosystem` means three different things across `/metrics` -- see the label sets above. **Ecosystem** filters the database-derived gauges, **Route** the request-path counters, and **Upstream** the two upstream fetch metrics. All three are query variables, so they populate from whatever labels your proxy is actually reporting; a panel is on the one its metric belongs to, and the panel descriptions say which. +Cache size and artifact count are refreshed every 60 seconds. The remaining metrics update on each request. ### Health Check @@ -1711,25 +860,12 @@ It carries three ecosystem filters rather than one, because `ecosystem` means th "checks": { "database": {"status": "ok"}, "storage": {"status": "ok"} - }, - "circuit_breakers": { - "registry.npmjs.org": "closed", - "static.crates.io": "open" - }, - "upstream_holds": { - "repo1.maven.org": "2026-10-14T11:15:29Z" } } ``` Failing checks include an `"error"` field. Storage failures also include a `"step"` field identifying which probe step failed (`write`, `size`, `read`, `verify`, `delete`). When the database check fails, the storage entry reports `{"status": "skipped"}` so the response always carries the same key set. -`circuit_breakers` reports the state of each upstream's artifact-fetch circuit breaker (`"open"` or `"closed"`), keyed by upstream host — or by the `hostless-url-` placeholder described under [Monitoring](#monitoring) where the fetch URL has no host to read. The key is omitted until the proxy has fetched an artifact from at least one upstream, and a host appears only once a breaker has been created for it. Breakers trip after repeated upstream failures and retry the upstream after an exponential backoff. While one is open, artifact downloads for that host return HTTP 502 on a cache miss without contacting the upstream; already-cached artifacts are still served from storage, since the cache is checked before the fetcher. A breaker is reported as `"open"` throughout its backoff, including the half-open window in which it admits one probe request to test recovery. Breaker state is per process and in memory, so a restart clears it, but a restart is not needed for recovery: the backoff keeps retrying for as long as the breaker is open, so it closes on its own once the upstream serves again. - -`upstream_holds` lists upstream hosts that answered the proxy with HTTP 429, or 503 with a `Retry-After`, and the time until which the proxy sends them nothing. Every request to a held host, metadata and artifacts alike, fails locally for the duration the upstream asked for (capped at one hour), or for one minute doubling up to fifteen when the upstream gave no `Retry-After`. Cached metadata is served stale during a hold, and anything uncached returns HTTP 503 with the remaining `Retry-After` so a client can pace itself too. Metadata requests do not go through the circuit breaker, so without the hold a rate-limited upstream would keep receiving one request per cache miss, which is exactly the traffic it is asking the proxy to stop sending. The first non-limited response after a hold ends clears it, and `proxy_upstream_rate_limit_holds_total` counts the holds placed. Holds are recorded in the database, so instances sharing one honour each other's holds and list them here too. The key is omitted while no host is held. - -An open breaker does **not** set `status` to `"error"` or change the HTTP status code: it reports a specific upstream refusing to serve, not this proxy being unfit to receive traffic, and failing the readiness probe over one unhealthy upstream would pull the pod out of rotation for every other ecosystem too. Use `proxy_circuit_breaker_state` for alerting on it. - Storage probe results are cached for `health.storage_probe_interval` (default 30s) to bound the cost of probing remote backends. A probe holds an internal mutex for up to 10 seconds (the hardcoded per-probe timeout), so `/health` is intended as a Kubernetes **readiness** probe rather than a liveness probe — a slow S3 round-trip should pull the pod from rotation, not restart it. Scrape config for Prometheus: @@ -1772,23 +908,13 @@ sudo systemctl start proxy ### Docker -Images for linux/amd64 and linux/arm64 are published to `ghcr.io/git-pkgs/proxy` for each release, tagged with the version and `latest`: - -```bash -docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest -``` - -To build the image from the repo instead: +A Dockerfile is included in the repo. Build and run: ```bash docker build -t proxy . docker run -p 8080:8080 -v proxy-data:/data proxy ``` -The image has a health check that requests `/health` on port 8080. It checks the database and storage, so a container reports unhealthy when either fails. If you change the port with `PROXY_LISTEN`, override it with `--health-cmd`. - -With `--read-only`, also pass `--tmpfs /tmp`: the proxy writes some uploads to a temporary file before reading them. - With Postgres and S3: ```bash @@ -1878,12 +1004,6 @@ cache/artifacts/ └── nginx-1.24.0-1.fc39.x86_64.rpm ``` -To delete one version's cached files, so the next request fetches them again, enable `cache_delete` and use the Delete buttons on the version page in the web UI, or the [cache API](docs/configuration.md#cache-delete-api): - -```bash -curl -X DELETE 'http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre?filename=guava-32.1.3-jre.jar' -``` - Cache metadata is stored in SQLite (default) or PostgreSQL. To clear a local cache: ```bash @@ -1897,7 +1017,7 @@ The proxy will recreate the database on next start. Requirements: -- Go (the project version is declared in `go.mod`) +- Go 1.25 or later ```bash git clone https://github.com/git-pkgs/proxy.git @@ -1913,4 +1033,4 @@ go test ./... ## License -[GPL-3.0-or-later](LICENSE). +GPL-3.0-or-later diff --git a/cmd/proxy/hosted.go b/cmd/proxy/hosted.go deleted file mode 100644 index 36a798d..0000000 --- a/cmd/proxy/hosted.go +++ /dev/null @@ -1,278 +0,0 @@ -package main - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "flag" - "fmt" - "io" - "net/http" - "net/url" - "os" - "path/filepath" - "strings" - "text/tabwriter" - "time" - - "github.com/git-pkgs/proxy/internal/handler" -) - -const ( - defaultProxyURL = "http://localhost:8080" - hostedTokenEnv = "PROXY_HOSTED_TOKEN" - hostedClientTimeout = 10 * time.Minute - - pushArgCount = 4 // ecosystem, name, version, file - listArgCount = 2 // ecosystem, name - deleteArgCount = 3 // ecosystem, name, version - deleteMaxArgCount = 4 // ecosystem, name, version, file for generic files - tablePadding = 2 -) - -// hostedArtifact mirrors the JSON the /api/hosted endpoints return. -type hostedArtifact struct { - Version string `json:"version"` - Filename string `json:"filename"` - SHA256 string `json:"sha256"` - Size int64 `json:"size"` - PushedAt time.Time `json:"pushed_at"` -} - -type hostedPushResponse struct { - Ecosystem string `json:"ecosystem"` - Name string `json:"name"` - hostedArtifact -} - -type hostedListResponse struct { - Ecosystem string `json:"ecosystem"` - Name string `json:"name"` - Artifacts []hostedArtifact `json:"artifacts"` -} - -type hostedCommand struct { - flags *flag.FlagSet - proxyURL *string - usage string -} - -func newHostedCommand(name, usage, summary string, getenv func(string) string) hostedCommand { - fs := flag.NewFlagSet(name, flag.ContinueOnError) - fs.SetOutput(io.Discard) - proxyURL := getenv("PROXY_URL") - if proxyURL == "" { - proxyURL = defaultProxyURL - } - cmd := hostedCommand{flags: fs, usage: usage} - cmd.proxyURL = fs.String("url", proxyURL, "Proxy URL (env PROXY_URL)") - fs.Usage = func() { - fmt.Fprintf(os.Stderr, "git-pkgs proxy - %s\n\nUsage: %s\n\nFlags:\n", summary, usage) - fs.SetOutput(os.Stderr) - fs.PrintDefaults() - } - return cmd -} - -// parse parses flags and checks for between minArgs and maxArgs positional -// arguments. -func (c hostedCommand) parse(args []string, minArgs, maxArgs int) ([]string, error) { - if err := c.flags.Parse(args); err != nil { - if errors.Is(err, flag.ErrHelp) { - c.flags.Usage() - } - return nil, fmt.Errorf("%w\nusage: %s", err, c.usage) - } - if n := c.flags.NArg(); n < minArgs || n > maxArgs { - return nil, fmt.Errorf("usage: %s", c.usage) - } - return c.flags.Args(), nil -} - -func (c hostedCommand) endpoint(ecosystem, name string, version ...string) string { - segments := []string{url.PathEscape(ecosystem)} - for segment := range strings.SplitSeq(name, "/") { - segments = append(segments, url.PathEscape(segment)) - } - for _, v := range version { - segments = append(segments, url.PathEscape(v)) - } - return strings.TrimSuffix(*c.proxyURL, "/") + "/api/hosted/" + strings.Join(segments, "/") -} - -func hostedToken(getenv func(string) string) (string, error) { - token := getenv(hostedTokenEnv) - if token == "" { - return "", fmt.Errorf("%s must be set to the proxy's hosted.token", hostedTokenEnv) - } - return token, nil -} - -func runPush(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { - cmd := newHostedCommand("push", "proxy hosted push [-url URL] ", - "Push a package archive to the proxy", getenv) - positional, err := cmd.parse(args, pushArgCount, pushArgCount) - if err != nil { - return err - } - token, err := hostedToken(getenv) - if err != nil { - return err - } - ecosystem, name, version, path := positional[0], positional[1], positional[2], positional[3] - - file, err := os.Open(path) - if err != nil { - return err - } - defer func() { _ = file.Close() }() - - // The proxy checks the upload against this, so a truncated or corrupted - // transfer is refused rather than stored. - digest := sha256.New() - size, err := io.Copy(digest, file) - if err != nil { - return err - } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - - segments := []string{version} - if handler.HostedFileInPath(ecosystem) { - segments = append(segments, filepath.Base(path)) - } - endpoint := cmd.endpoint(ecosystem, name, segments...) + "?sha256=" + hex.EncodeToString(digest.Sum(nil)) - req, err := http.NewRequest(http.MethodPut, endpoint, file) - if err != nil { - return err - } - req.ContentLength = size - req.Header.Set("Authorization", "Bearer "+token) - req.Header.Set("Content-Type", "application/octet-stream") - // The proxy refuses an existing file or a bad token before reading the - // body, so the file is only uploaded once it would be accepted. - req.Header.Set("Expect", "100-continue") - - var pushed hostedPushResponse - if err := doHosted(client, req, http.StatusCreated, &pushed); err != nil { - return err - } - _, err = fmt.Fprintf(stdout, "pushed %s %s@%s as %s (%d bytes, sha256 %s)\n", - pushed.Ecosystem, pushed.Name, pushed.Version, pushed.Filename, pushed.Size, pushed.SHA256) - return err -} - -func runList(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { - cmd := newHostedCommand("list", "proxy hosted list [-url URL] ", "List pushed package archives", getenv) - positional, err := cmd.parse(args, listArgCount, listArgCount) - if err != nil { - return err - } - req, err := http.NewRequest(http.MethodGet, cmd.endpoint(positional[0], positional[1]), nil) - if err != nil { - return err - } - - var listed hostedListResponse - if err := doHosted(client, req, http.StatusOK, &listed); err != nil { - return err - } - tw := tabwriter.NewWriter(stdout, 0, 0, tablePadding, ' ', 0) - _, _ = fmt.Fprintln(tw, "VERSION\tFILENAME\tSIZE\tSHA256\tPUSHED") - for _, a := range listed.Artifacts { - _, _ = fmt.Fprintf(tw, "%s\t%s\t%d\t%s\t%s\n", a.Version, a.Filename, a.Size, a.SHA256, a.PushedAt.Format(time.RFC3339)) - } - return tw.Flush() -} - -func runDelete(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { - cmd := newHostedCommand("delete", "proxy hosted delete [-url URL] [file]", - "Delete a pushed version, or one generic file of it", getenv) - positional, err := cmd.parse(args, deleteArgCount, deleteMaxArgCount) - if err != nil { - return err - } - if len(positional) > deleteArgCount && !handler.HostedFileInPath(positional[0]) { - return fmt.Errorf("only generic files can be deleted one at a time\nusage: %s", cmd.usage) - } - token, err := hostedToken(getenv) - if err != nil { - return err - } - req, err := http.NewRequest(http.MethodDelete, cmd.endpoint(positional[0], positional[1], positional[2:]...), nil) - if err != nil { - return err - } - req.Header.Set("Authorization", "Bearer "+token) - - if err := doHosted(client, req, http.StatusNoContent, nil); err != nil { - return err - } - deleted := positional[1] + "@" + positional[2] - if len(positional) > deleteArgCount { - deleted += " " + positional[deleteArgCount] - } - _, err = fmt.Fprintf(stdout, "deleted %s %s\n", positional[0], deleted) - return err -} - -// doHosted sends req and decodes a successful response into out. Any other -// status is returned as an error carrying the server's message. -func doHosted(client *http.Client, req *http.Request, wantStatus int, out any) error { - resp, err := client.Do(req) - if err != nil { - return err - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != wantStatus { - var apiErr struct { - Message string `json:"message"` - } - if json.NewDecoder(resp.Body).Decode(&apiErr) == nil && apiErr.Message != "" { - return fmt.Errorf("%s %s: %s (%s)", req.Method, req.URL.Path, apiErr.Message, resp.Status) - } - return fmt.Errorf("%s %s: %s", req.Method, req.URL.Path, resp.Status) - } - if out == nil { - return nil - } - return json.NewDecoder(resp.Body).Decode(out) -} - -const hostedUsage = "usage: proxy hosted [flags] ...\n\n" + - " push Push a package archive to the proxy\n" + - " list List pushed package archives\n" + - " delete Delete a pushed version, or one generic file of it" - -type hostedSubcommand func([]string, func(string) string, io.Writer, *http.Client) error - -var hostedSubcommands = map[string]hostedSubcommand{ - "push": runPush, - "list": runList, - "delete": runDelete, -} - -// runHosted runs the hosted subcommand named by args[0]. -func runHosted(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { - if len(args) == 0 { - return errors.New(hostedUsage) - } - run, ok := hostedSubcommands[args[0]] - if !ok { - return fmt.Errorf("unknown hosted command %q\n%s", args[0], hostedUsage) - } - return run(args[1:], getenv, stdout, client) -} - -// runHostedCommand runs proxy hosted against the proxy's /api/hosted -// endpoints and exits non-zero on failure. -func runHostedCommand() { - client := &http.Client{Timeout: hostedClientTimeout} - if err := runHosted(os.Args[1:], os.Getenv, os.Stdout, client); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} diff --git a/cmd/proxy/hosted_test.go b/cmd/proxy/hosted_test.go deleted file mode 100644 index 33fe226..0000000 --- a/cmd/proxy/hosted_test.go +++ /dev/null @@ -1,220 +0,0 @@ -package main - -import ( - "bytes" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "testing" -) - -type recordedRequest struct { - method, path, query, authorization, body, expect string - contentLength int64 -} - -func hostedTestServer(t *testing.T, status int, response string) (*httptest.Server, *recordedRequest) { - t.Helper() - recorded := &recordedRequest{} - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - *recorded = recordedRequest{ - method: r.Method, - path: r.URL.EscapedPath(), - query: r.URL.RawQuery, - authorization: r.Header.Get("Authorization"), - body: string(body), - expect: r.Header.Get("Expect"), - contentLength: r.ContentLength, - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _, _ = io.WriteString(w, response) - })) - t.Cleanup(srv.Close) - return srv, recorded -} - -func hostedEnv(vars map[string]string) func(string) string { - return func(key string) string { return vars[key] } -} - -func writeArchive(t *testing.T, name, content string) string { - t.Helper() - path := filepath.Join(t.TempDir(), name) - if err := os.WriteFile(path, []byte(content), 0o600); err != nil { - t.Fatalf("writing archive: %v", err) - } - return path -} - -func TestRunPush(t *testing.T) { - srv, recorded := hostedTestServer(t, http.StatusCreated, - `{"ecosystem":"npm","name":"@types/node","version":"20.1.0","filename":"node-20.1.0.tgz","sha256":"abc123","size":7}`) - archive := writeArchive(t, "node-20.1.0.tgz", "tarball") - var stdout bytes.Buffer - - err := runPush([]string{"-url", srv.URL, "npm", "@types/node", "20.1.0", archive}, - hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client()) - if err != nil { - t.Fatalf("runPush: %v", err) - } - if recorded.method != http.MethodPut || recorded.path != "/api/hosted/npm/@types/node/20.1.0" { - t.Errorf("request = %s %s", recorded.method, recorded.path) - } - // sha256 of "tarball", so the proxy can check the upload. - const tarballSHA256 = "db4b4d0d1cb480bf9aeea253771c00febe627f236765fa37d6a5614f079a3aa0" - if recorded.authorization != "Bearer tok" || recorded.body != "tarball" || recorded.query != "sha256="+tarballSHA256 { - t.Errorf("authorization/body/query = %q/%q/%q", recorded.authorization, recorded.body, recorded.query) - } - if recorded.expect != "100-continue" || recorded.contentLength != int64(len("tarball")) { - t.Errorf("expect/content length = %q/%d, want 100-continue and the file size", recorded.expect, recorded.contentLength) - } - if out := stdout.String(); !strings.Contains(out, "node-20.1.0.tgz") || !strings.Contains(out, "abc123") { - t.Errorf("output = %q, want filename and sha256", out) - } -} - -func TestRunPushGenericNamesTheFile(t *testing.T) { - srv, recorded := hostedTestServer(t, http.StatusCreated, - `{"ecosystem":"generic","name":"tools","version":"1.2.0","filename":"tool-linux-arm64.tar.gz","sha256":"abc123","size":7}`) - archive := writeArchive(t, "tool-linux-arm64.tar.gz", "tarball") - - err := runPush([]string{"-url", srv.URL, "generic", "tools", "1.2.0", archive}, - hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) - if err != nil { - t.Fatalf("runPush: %v", err) - } - if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" { - t.Errorf("path = %q, want the file name appended", recorded.path) - } -} - -func TestRunPushErrors(t *testing.T) { - srv, _ := hostedTestServer(t, http.StatusBadRequest, `{"code":"BAD_REQUEST","message":"pushing packages is not supported for this ecosystem: cargo"}`) - archive := writeArchive(t, "serde-1.0.200.crate", "crate") - - err := runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive}, hostedEnv(nil), io.Discard, srv.Client()) - if err == nil || !strings.Contains(err.Error(), "PROXY_HOSTED_TOKEN") { - t.Errorf("push without token error = %v, want a PROXY_HOSTED_TOKEN hint", err) - } - - err = runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive}, - hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) - if err == nil || !strings.Contains(err.Error(), "not supported for this ecosystem") { - t.Errorf("rejected push error = %v, want the server's message", err) - } - - err = runPush([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) - if err == nil || !strings.Contains(err.Error(), "usage") { - t.Errorf("push with missing arguments error = %v, want usage", err) - } -} - -func TestRunPushUsesProxyURLFromEnv(t *testing.T) { - srv, recorded := hostedTestServer(t, http.StatusCreated, `{}`) - archive := writeArchive(t, "request-2.88.3.tgz", "tarball") - - err := runPush([]string{"npm", "request", "2.88.3", archive}, - hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok", "PROXY_URL": srv.URL}), io.Discard, srv.Client()) - if err != nil { - t.Fatalf("runPush: %v", err) - } - if recorded.path != "/api/hosted/npm/request/2.88.3" { - t.Errorf("path = %q", recorded.path) - } -} - -func TestRunList(t *testing.T) { - srv, recorded := hostedTestServer(t, http.StatusOK, - `{"ecosystem":"npm","name":"request","artifacts":[{"version":"2.88.3","filename":"request-2.88.3.tgz","sha256":"abc123","size":7,"pushed_at":"2026-10-13T10:00:00Z"}]}`) - var stdout bytes.Buffer - - if err := runList([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(nil), &stdout, srv.Client()); err != nil { - t.Fatalf("runList: %v", err) - } - if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" || recorded.authorization != "" { - t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization) - } - if out := stdout.String(); !strings.Contains(out, "2.88.3") || !strings.Contains(out, "request-2.88.3.tgz") { - t.Errorf("output = %q", out) - } -} - -func TestRunDelete(t *testing.T) { - srv, recorded := hostedTestServer(t, http.StatusNoContent, "") - - err := runDelete([]string{"-url", srv.URL, "npm", "request", "2.88.3"}, - hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) - if err != nil { - t.Fatalf("runDelete: %v", err) - } - if recorded.method != http.MethodDelete || recorded.path != "/api/hosted/npm/request/2.88.3" || recorded.authorization != "Bearer tok" { - t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization) - } - - missing, _ := hostedTestServer(t, http.StatusNotFound, `{"code":"NOT_FOUND","message":"no pushed artifacts for this version"}`) - err = runDelete([]string{"-url", missing.URL, "npm", "request", "2.88.3"}, - hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, missing.Client()) - if err == nil || !strings.Contains(err.Error(), "no pushed artifacts") { - t.Errorf("delete of a missing version error = %v", err) - } -} - -func TestRunDeleteOneFile(t *testing.T) { - srv, recorded := hostedTestServer(t, http.StatusNoContent, "") - var stdout bytes.Buffer - - err := runDelete([]string{"-url", srv.URL, "generic", "tools", "1.2.0", "tool-linux-arm64.tar.gz"}, - hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client()) - if err != nil { - t.Fatalf("runDelete: %v", err) - } - if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" { - t.Errorf("path = %q", recorded.path) - } - if out := stdout.String(); out != "deleted generic tools@1.2.0 tool-linux-arm64.tar.gz\n" { - t.Errorf("output = %q", out) - } -} - -func TestRunDeleteOneFileOnlyForGeneric(t *testing.T) { - srv, recorded := hostedTestServer(t, http.StatusNoContent, "") - - err := runDelete([]string{"-url", srv.URL, "npm", "@acme", "widget", "1.0.0"}, - hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) - if err == nil || !strings.Contains(err.Error(), "only generic files") { - t.Errorf("npm delete with a fourth argument error = %v, want a refusal", err) - } - if recorded.method != "" { - t.Errorf("refused delete sent %s %s", recorded.method, recorded.path) - } -} - -func TestRunHostedDispatchesSubcommands(t *testing.T) { - srv, recorded := hostedTestServer(t, http.StatusOK, `{"ecosystem":"npm","name":"request","artifacts":[]}`) - - if err := runHosted([]string{"list", "-url", srv.URL, "npm", "request"}, hostedEnv(nil), io.Discard, srv.Client()); err != nil { - t.Fatalf("runHosted list: %v", err) - } - if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" { - t.Errorf("request = %s %s", recorded.method, recorded.path) - } - - for _, args := range [][]string{nil, {"bogus"}} { - err := runHosted(args, hostedEnv(nil), io.Discard, srv.Client()) - if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted ") { - t.Errorf("runHosted(%q) error = %v, want the hosted usage", args, err) - } - } -} - -func TestHostedSubcommandUsageNamesTheHostedCommand(t *testing.T) { - err := runPush([]string{"npm"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, http.DefaultClient) - if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted push") { - t.Errorf("push usage error = %v, want it to name proxy hosted push", err) - } -} diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 7c51732..15a71c0 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -5,11 +5,6 @@ // @description HTTP API for package enrichment, vulnerability lookup, cache stats, and source browsing. // @BasePath / // -// @securityDefinitions.apikey BearerAuth -// @in header -// @name Authorization -// @description "Bearer " followed by hosted.token, or projects.token for /api/projects -// // The proxy caches package artifacts from upstream registries (npm, cargo, etc.) // providing faster, more reliable access for development teams. // @@ -22,7 +17,6 @@ // serve Start the proxy server (default if no command given) // stats Show cache statistics // mirror Pre-populate cache from PURLs, SBOMs, or registries -// hosted Push, list or delete packages on a running proxy (needs hosted.enabled) // // Serve Flags: // @@ -46,8 +40,6 @@ // Log level: debug, info, warn, error (default "info") // -log-format string // Log format: text, json (default "text") -// -access-log string -// Path to the JSONL access log (disabled by default) // // Stats Flags: // @@ -80,23 +72,14 @@ // PROXY_DATABASE_URL - PostgreSQL connection URL // PROXY_LOG_LEVEL - Log level // PROXY_LOG_FORMAT - Log format -// PROXY_ACCESS_LOG_PATH - JSONL access log path -// PROXY_UPSTREAM_* - Upstream URLs and network access controls -// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Set false to allow Gradle PUT uploads (default true) +// PROXY_UPSTREAM_MAVEN - Maven repository upstream URL +// PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL - Gradle Plugin Portal upstream URL +// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Disable Gradle PUT uploads // PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE - Max Gradle PUT request body size // PROXY_GRADLE_BUILD_CACHE_MAX_AGE - Gradle cache max age eviction // PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size // PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL - Gradle cache eviction sweep interval // PROXY_HEALTH_STORAGE_PROBE_INTERVAL - Storage health probe cache interval (default "30s") -// PROXY_HOSTED_ENABLED - Enable pushing packages to /api/hosted -// PROXY_HOSTED_TOKEN - Bearer token for pushing and deleting -// PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB") -// PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only -// PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only -// PROXY_PROJECTS_ENABLED - Enable the project inventory (/api/projects, /ui/projects) -// PROXY_PROJECTS_TOKEN - Bearer token for uploading and deleting project SBOMs -// PROXY_PROJECTS_MAX_AGE - Remove projects not uploaded for this long (e.g. "90d") -// PROXY_URL - Proxy URL for proxy hosted // // Example: // @@ -121,17 +104,16 @@ import ( "log/slog" "os" "os/signal" - "runtime/debug" "strings" "syscall" "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" - "github.com/git-pkgs/proxy/internal/memlimit" "github.com/git-pkgs/proxy/internal/mirror" "github.com/git-pkgs/proxy/internal/server" "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" ) const defaultTopN = 10 @@ -144,15 +126,6 @@ var ( Commit = "unknown" ) -func init() { - if Version != "dev" { - return - } - if bi, ok := debug.ReadBuildInfo(); ok && bi.Main.Version != "" && bi.Main.Version != "(devel)" { - Version = bi.Main.Version - } -} - func main() { if len(os.Args) > 1 { switch os.Args[1] { @@ -168,10 +141,6 @@ func main() { os.Args = append(os.Args[:1], os.Args[2:]...) runMirror() return - case "hosted": - os.Args = append(os.Args[:1], os.Args[2:]...) - runHostedCommand() - return case "-version", "--version": fmt.Printf("proxy %s (%s)\n", Version, Commit) os.Exit(0) @@ -194,7 +163,6 @@ Commands: serve Start the proxy server (default) stats Show cache statistics mirror Pre-populate cache from PURLs, SBOMs, or registries - hosted Push, list or delete packages on a running proxy Run 'proxy -help' for more information on a command. @@ -216,7 +184,6 @@ func runServe() { databaseURL := fs.String("database-url", "", "PostgreSQL connection URL") logLevel := fs.String("log-level", "", "Log level: debug, info, warn, error") logFormat := fs.String("log-format", "", "Log format: text, json") - accessLogPath := fs.String("access-log", "", "Path to the JSONL access log") version := fs.Bool("version", false, "Print version and exit") fs.Usage = func() { @@ -234,36 +201,9 @@ func runServe() { fmt.Fprintf(os.Stderr, " PROXY_DATABASE_URL PostgreSQL connection URL\n") fmt.Fprintf(os.Stderr, " PROXY_LOG_LEVEL Log level\n") fmt.Fprintf(os.Stderr, " PROXY_LOG_FORMAT Log format\n") - fmt.Fprintf(os.Stderr, " PROXY_ACCESS_LOG_PATH JSONL access log path\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS Comma-separated private upstream hosts\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_ALLOW_LOOPBACK Permit loopback upstreams and redirects\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_NPM npm registry upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CARGO Cargo index upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CARGO_DOWNLOAD Cargo download upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GEM RubyGems upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GO Go module proxy upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GO_VULNDB Go vulnerability database URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_HEX Hex repository upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_HEX_API Hex API upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_PUB pub registry upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_PYPI PyPI index and API upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_PYPI_DOWNLOAD PyPI download upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_MAVEN Maven repository upstream URL\n") + fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_MAVEN Maven repository upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL Gradle Plugin Portal upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_NUGET NuGet API upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_NUGET_SEARCH NuGet search upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER Packagist API upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER_REPOSITORY Packagist repository upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONAN Conan registry upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CHEF Chef Supermarket upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONDA Conda channel upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CRAN CRAN mirror upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_JULIA Julia package server upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_SWIFT Swift Package Registry upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_OCI_DEFAULT Default OCI registry upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_DEBIAN Debian repository upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_RPM RPM repository upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Set false to allow Gradle PUT uploads (default true)\n") + fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Disable Gradle PUT uploads\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE Max Gradle PUT request body size\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_AGE Gradle cache max age eviction\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_SIZE Gradle cache max total size\n") @@ -316,9 +256,6 @@ func runServe() { if *logFormat != "" { cfg.Log.Format = *logFormat } - if *accessLogPath != "" { - cfg.AccessLog.Path = *accessLogPath - } // Validate configuration if err := cfg.Validate(); err != nil { @@ -328,13 +265,9 @@ func runServe() { // Setup logger logger := setupLogger(cfg.Log.Level, cfg.Log.Format) - memlimit.Apply(logger) // Create and start server - srv, err := server.New(cfg, logger, server.BuildInfo{ - Version: Version, - Commit: Commit, - }) + srv, err := server.New(cfg, logger) if err != nil { logger.Error("failed to create server", "error", err) os.Exit(1) @@ -443,8 +376,6 @@ func runMirror() { sbomPath := fs.String("sbom", "", "Path to CycloneDX or SPDX SBOM file") concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading") - platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored") - latest := fs.Bool("latest", false, "Mirror only the newest version of PURLs without a version") fs.Usage = func() { fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n") @@ -452,36 +383,21 @@ func runMirror() { fmt.Fprintf(os.Stderr, "Examples:\n") fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash@4.17.21\n") fmt.Fprintf(os.Stderr, " proxy mirror --sbom sbom.cdx.json\n") - fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n") - fmt.Fprintf(os.Stderr, " proxy mirror --latest pkg:npm/lodash # newest version\n") - fmt.Fprintf(os.Stderr, " proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n") + fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n\n") fmt.Fprintf(os.Stderr, "Flags:\n") fs.PrintDefaults() } _ = fs.Parse(os.Args[1:]) purls := fs.Args() - var platformList []string - if *platforms != "" { - platformList = []string{*platforms} - } - if _, err := mirror.ParsePlatforms(platformList); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } // Determine source var source mirror.Source switch { case *sbomPath != "": - data, err := os.ReadFile(*sbomPath) - if err != nil { - fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err) - os.Exit(1) - } - source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Latest: *latest, Platforms: platformList} + source = &mirror.SBOMSource{Path: *sbomPath} case len(purls) > 0: - source = &mirror.PURLSource{PURLs: purls, Latest: *latest, Platforms: platformList} + source = &mirror.PURLSource{PURLs: purls} default: fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n") fs.Usage() @@ -495,7 +411,6 @@ func runMirror() { os.Exit(1) } cfg.LoadFromEnv() - printEnvWarnings(cfg) if *storageURL != "" { cfg.Storage.URL = *storageURL @@ -514,10 +429,6 @@ func runMirror() { fmt.Fprintf(os.Stderr, "invalid configuration: %v\n", err) os.Exit(1) } - if !cfg.Storage.CacheArtifacts { - fmt.Fprintf(os.Stderr, "error: mirror is not available with storage.cache_artifacts: false: mirrored artifacts would never be served\n") - os.Exit(1) - } logger := setupLogger("info", "text") @@ -553,27 +464,15 @@ func runMirror() { os.Exit(1) //nolint:gocritic // db closed above } - // Build proxy with the same upstream clients and authentication as serve - proxy, images := server.NewMirrorProxy(cfg, db, store, logger) + // Build proxy (reuses same pipeline as serve) + fetcher := fetch.NewFetcher() + resolver := fetch.NewResolver() + proxy := handler.NewProxy(db, store, fetcher, resolver, logger) proxy.CacheMetadata = true // mirror always caches metadata proxy.MetadataTTL = cfg.ParseMetadataTTL() proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() - proxy.MetadataStreamMaxSize = cfg.ParseMetadataStreamMaxSize() - // As serve does, resolve Swift source repositories in the denylist through - // the upstream, and refuse to run when one can't be. - if err := server.ConfigureDenylist(cfg, proxy); err != nil { - _ = db.Close() - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) //nolint:gocritic // db closed above - } - m := newCommandMirror(cfg, proxy, images, db, store, logger, *concurrency) - switch s := source.(type) { - case *mirror.PURLSource: - s.Versions = m.Versions - case *mirror.SBOMSource: - s.Versions = m.Versions - } + m := mirror.New(proxy, db, store, logger, *concurrency) ctx, cancel := context.WithCancel(context.Background()) go func() { @@ -596,9 +495,7 @@ func runMirror() { return } - closeGit := useCommandGit(cfg, proxy, m, logger) progress, err := m.Run(ctx, source) - closeGit() if err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) @@ -613,75 +510,6 @@ func runMirror() { fmt.Fprintf(os.Stderr, " %s/%s@%s: %s\n", e.Ecosystem, e.Name, e.Version, e.Error) } } - - // A partial mirror has to be visible in the exit status, or a CI step - // that prewarms the cache passes even when every download failed. - if code := mirrorExitCode(progress); code != 0 { - _ = db.Close() - os.Exit(code) //nolint:gocritic // db closed above - } -} - -// newCommandMirror builds the mirror proxy mirror runs, which fetches and -// lists versions from the configured upstreams. -func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler.ContainerHandler, - db *database.DB, store storage.Storage, logger *slog.Logger, workers int) *mirror.Mirror { - m := mirror.New(proxy, db, store, logger, workers) - m.Images = images - m.Bottles = handler.NewHomebrewHandler(proxy, cfg.Upstream.HomebrewAPI) - m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm) - server.UseMirrorUpstreams(m, cfg.Upstream, proxy.HTTPClient) - if cfg.Scanning.Enabled { - refuseUnscannedMirroring(m) - } - return m -} - -// useCommandGit opens the git cache for the pkg:git items of proxy mirror, -// when upstream.git is configured, and returns the function that closes it. -// Where it can't be opened, such as while the running server holds -// git.cache_dir, those items fail with the reason and the rest are mirrored. -func useCommandGit(cfg *config.Config, proxy *handler.Proxy, m *mirror.Mirror, logger *slog.Logger) func() { - git, closeGit, err := server.NewMirrorGit(cfg, proxy, logger) - if err != nil { - m.Git = mirror.UnavailableGit{Reason: fmt.Sprintf("this command can't open the git cache (%v); while the server runs, use its mirror API", err)} - return func() {} - } - if git == nil { - return func() {} - } - m.Git = git - return func() { - if err := closeGit(); err != nil { - logger.Warn("closing the git cache", "error", err) - } - } -} - -// mirrorExitCode returns the exit status for a finished mirror run: 1 when any -// package version failed to mirror, 0 otherwise. -func mirrorExitCode(progress *mirror.Progress) int { - if progress != nil && progress.Failed > 0 { - return 1 - } - return 0 -} - -// Reasons the mirror command gives for refusing items while scanning is -// enabled. -const ( - mirrorScanningImagesReason = "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server" - mirrorScanningPackagesReason = "packages cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server" -) - -// refuseUnscannedMirroring makes m fail every item instead of caching it. -// Scanners fetch the staged bytes from the running server, which this command -// does not start, so anything it cached would skip the pre-cache scan and -// later be served from the cache unscanned. The mirror API's jobs run inside -// the server with its scanners and are unaffected. -func refuseUnscannedMirroring(m *mirror.Mirror) { - m.Images = mirror.UnavailableImages{Reason: mirrorScanningImagesReason} - m.RefusePackages = mirrorScanningPackagesReason } func printStats(db *database.DB, popular, recent int, asJSON bool) error { @@ -826,14 +654,6 @@ func formatSize(bytes int64) string { } } -// printEnvWarnings reports environment values LoadFromEnv ignored. serve -// logs them from server.Start; mirror never gets there. -func printEnvWarnings(cfg *config.Config) { - for _, warning := range cfg.EnvWarnings() { - fmt.Fprintf(os.Stderr, "warning: %s\n", warning) - } -} - func loadConfig(path string) (*config.Config, error) { if path != "" { return config.Load(path) diff --git a/cmd/proxy/main_test.go b/cmd/proxy/main_test.go deleted file mode 100644 index d5a020d..0000000 --- a/cmd/proxy/main_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "os" - "os/exec" - "strings" - "testing" -) - -func TestServeHelpListsUpstreamEnvironmentVariables(t *testing.T) { - cmd := exec.Command(os.Args[0], "-test.run=^TestServeHelpProcess$") - cmd.Env = append(os.Environ(), "PROXY_TEST_SERVE_HELP=1") - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("serve help failed: %v\n%s", err, output) - } - - variables := []string{ - "PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS", - "PROXY_UPSTREAM_ALLOW_LOOPBACK", - "PROXY_UPSTREAM_NPM", - "PROXY_UPSTREAM_CARGO", - "PROXY_UPSTREAM_CARGO_DOWNLOAD", - "PROXY_UPSTREAM_GEM", - "PROXY_UPSTREAM_GO", - "PROXY_UPSTREAM_HEX", - "PROXY_UPSTREAM_HEX_API", - "PROXY_UPSTREAM_PUB", - "PROXY_UPSTREAM_PYPI", - "PROXY_UPSTREAM_PYPI_DOWNLOAD", - "PROXY_UPSTREAM_MAVEN", - "PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL", - "PROXY_UPSTREAM_NUGET", - "PROXY_UPSTREAM_NUGET_SEARCH", - "PROXY_UPSTREAM_COMPOSER", - "PROXY_UPSTREAM_COMPOSER_REPOSITORY", - "PROXY_UPSTREAM_CONAN", - "PROXY_UPSTREAM_CHEF", - "PROXY_UPSTREAM_CONDA", - "PROXY_UPSTREAM_CRAN", - "PROXY_UPSTREAM_JULIA", - "PROXY_UPSTREAM_OCI_DEFAULT", - "PROXY_UPSTREAM_DEBIAN", - "PROXY_UPSTREAM_RPM", - } - for _, variable := range variables { - if !strings.Contains(string(output), variable) { - t.Errorf("serve help omitted %s", variable) - } - } -} - -func TestServeHelpProcess(*testing.T) { - if os.Getenv("PROXY_TEST_SERVE_HELP") != "1" { - return - } - os.Args = []string{"proxy", "serve", "-help"} - main() -} - -func TestMirrorLogsEnvWarnings(t *testing.T) { - cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorEnvWarningsProcess$") - // The negative size stops mirror before it opens any storage. - cmd.Env = append(os.Environ(), - "PROXY_TEST_MIRROR_ENV_WARNINGS=1", - "PROXY_CACHE_METADATA=maybe", - "PROXY_STORAGE_MAX_SIZE=-1") - output, _ := cmd.CombinedOutput() - if !strings.Contains(string(output), "PROXY_CACHE_METADATA") { - t.Errorf("mirror did not warn about PROXY_CACHE_METADATA:\n%s", output) - } -} - -func TestMirrorEnvWarningsProcess(*testing.T) { - if os.Getenv("PROXY_TEST_MIRROR_ENV_WARNINGS") != "1" { - return - } - os.Args = []string{"proxy", "mirror", "pkg:npm/lodash@4.17.21"} - main() -} diff --git a/cmd/proxy/mirror_git_test.go b/cmd/proxy/mirror_git_test.go deleted file mode 100644 index 2789536..0000000 --- a/cmd/proxy/mirror_git_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package main - -import ( - "net/http" - "net/http/cgi" - "net/http/httptest" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/gitcache" -) - -// gitMirrorUpstream serves team/repo, with one commit on main, from git -// http-backend, and returns the server and the commit. -func gitMirrorUpstream(t *testing.T) (*httptest.Server, string) { - t.Helper() - git := func(dir string, args ...string) string { - t.Helper() - cmd := exec.Command("git", args...) - cmd.Dir = dir - cmd.Env = []string{ - "PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", - "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", - "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", - } - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) - } - return strings.TrimSpace(string(out)) - } - backend := "" - for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { - if path := filepath.Join(git("", "--exec-path"), name); fileExistsForTest(path) { - backend = path - } - } - if backend == "" { - t.Skip("git http-backend is not installed") - } - root := t.TempDir() - bare := filepath.Join(root, "team", "repo.git") - git("", "init", "--quiet", "--bare", "--initial-branch=main", bare) - work := t.TempDir() - git("", "clone", "--quiet", bare, work) - git(work, "commit", "--quiet", "--allow-empty", "-m", "first") - git(work, "push", "--quiet", "origin", "HEAD:main") - head := git(work, "rev-parse", "HEAD") - - server := httptest.NewServer(&cgi.Handler{ - Path: backend, - Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}, - }) - t.Cleanup(server.Close) - return server, head -} - -func fileExistsForTest(path string) bool { - _, err := os.Stat(path) - return err == nil -} - -// writeGitMirrorConfig writes a config with upstream.git "up" at -// upstreamURL and the git cache, storage and database under dir. -func writeGitMirrorConfig(t *testing.T, dir, upstreamURL string) string { - t.Helper() - cfg := "storage:\n" + - " url: file://" + filepath.Join(dir, "artifacts") + "\n" + - "database:\n" + - " path: " + filepath.Join(dir, "proxy.db") + "\n" + - "upstream:\n" + - " allow_loopback: true\n" + - " git:\n" + - " up: " + upstreamURL + "\n" + - "git:\n" + - " cache_dir: " + filepath.Join(dir, "git") + "\n" - path := filepath.Join(dir, "config.yaml") - if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil { - t.Fatalf("writing config: %v", err) - } - return path -} - -func TestMirrorCommandCachesGitCommit(t *testing.T) { - upstream, head := gitMirrorUpstream(t) - dir := t.TempDir() - configPath := writeGitMirrorConfig(t, dir, upstream.URL) - purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head - - output := runMirrorProcess(t, 0, "-config", configPath, purl) - if !strings.Contains(output, "1 downloaded") { - t.Errorf("output:\n%s", output) - } - cached, _ := filepath.Glob(filepath.Join(dir, "git", "up-*", "team", "repo.git")) - if len(cached) != 1 { - t.Errorf("cached repositories %v, want team/repo.git", cached) - } - - output = runMirrorProcess(t, 0, "-config", configPath, purl) - if !strings.Contains(output, "1 skipped") { - t.Errorf("second run output:\n%s", output) - } -} - -// TestMirrorCommandGitCacheInUse checks pkg:git items fail, pointing at the -// mirror API, while the running server holds git.cache_dir. -func TestMirrorCommandGitCacheInUse(t *testing.T) { - upstream, head := gitMirrorUpstream(t) - dir := t.TempDir() - configPath := writeGitMirrorConfig(t, dir, upstream.URL) - cache, err := gitcache.New(gitcache.Options{ - CacheDir: filepath.Join(dir, "git"), - RefreshWait: time.Second, - Client: &http.Client{}, - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = cache.Close() }() - purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head - - output := runMirrorProcess(t, 1, "-config", configPath, purl) - for _, want := range []string{"in use by another process", "mirror API", "1 failed"} { - if !strings.Contains(output, want) { - t.Errorf("output missing %q:\n%s", want, output) - } - } -} diff --git a/cmd/proxy/mirror_swift_test.go b/cmd/proxy/mirror_swift_test.go deleted file mode 100644 index 28fa8ba..0000000 --- a/cmd/proxy/mirror_swift_test.go +++ /dev/null @@ -1,186 +0,0 @@ -package main - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "sync/atomic" - "testing" - - "github.com/git-pkgs/proxy/internal/database" -) - -const swiftWidgetArchive = "acme widget archive" - -// swiftMirrorUpstream serves acme.widget 1.0.0 the way a Swift package -// registry does, with https://github.com/acme/widget as its source -// repository, and counts archive downloads and identifier lookups. The -// repository's identifiers are ids, or only acme.widget. -func swiftMirrorUpstream(t *testing.T, ids ...string) (srv *httptest.Server, downloads, lookups *atomic.Int64) { - t.Helper() - if len(ids) == 0 { - ids = []string{"acme.widget"} - } - identifiers, err := json.Marshal(map[string][]string{"identifiers": ids}) - if err != nil { - t.Fatal(err) - } - downloads, lookups = new(atomic.Int64), new(atomic.Int64) - sum := sha256.Sum256([]byte(swiftWidgetArchive)) - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Version", "1") - switch r.URL.Path { - case "/identifiers": - lookups.Add(1) - if r.URL.Query().Get("url") != "https://github.com/acme/widget" { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(identifiers) - case "/acme/widget": - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"releases":{"1.0.0":{}}}`) - case "/acme/widget/1.0.0/Package.swift": - w.Header().Set("Content-Type", "text/x-swift") - _, _ = io.WriteString(w, "// swift-tools-version:5.9\n") - case "/acme/widget/1.0.0": - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"id":"acme.widget","version":"1.0.0","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(sum[:])) - case "/acme/widget/1.0.0.zip": - downloads.Add(1) - w.Header().Set("Content-Type", "application/zip") - _, _ = io.WriteString(w, swiftWidgetArchive) - default: - http.NotFound(w, r) - } - })) - t.Cleanup(srv.Close) - return srv, downloads, lookups -} - -// writeSwiftMirrorConfig writes a config using upstream as the Swift -// registry, denying the denied package PURLs. -func writeSwiftMirrorConfig(t *testing.T, dir, upstream string, denied ...string) string { - t.Helper() - cfg := "storage:\n" + - " url: file://" + filepath.Join(dir, "artifacts") + "\n" + - "database:\n" + - " path: " + filepath.Join(dir, "proxy.db") + "\n" + - "upstream:\n" + - " swift: " + upstream + "\n" + - " allow_loopback: true\n" - if len(denied) > 0 { - cfg += "denylist:\n packages:\n" - for _, p := range denied { - cfg += " - " + p + "\n" - } - } - path := filepath.Join(dir, "config.yaml") - if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil { - t.Fatalf("writing config: %v", err) - } - return path -} - -func cachedArtifacts(t *testing.T, dir string) int64 { - t.Helper() - db, err := database.Open(filepath.Join(dir, "proxy.db")) - if err != nil { - t.Fatalf("opening database: %v", err) - } - defer func() { _ = db.Close() }() - stats, err := db.GetCacheStats() - if err != nil { - t.Fatalf("GetCacheStats: %v", err) - } - return stats.TotalArtifacts -} - -// Without a denylist the fake registry mirrors the release, so the tests -// below fail for the denylist and not for the setup. -func TestMirrorSwiftRelease(t *testing.T) { - upstream, downloads, _ := swiftMirrorUpstream(t) - dir := t.TempDir() - configPath := writeSwiftMirrorConfig(t, dir, upstream.URL) - - output := runMirrorProcess(t, 0, "-config", configPath, "pkg:swift/acme/widget@1.0.0") - - if !strings.Contains(output, "1 downloaded") { - t.Errorf("output:\n%s", output) - } - if n := downloads.Load(); n != 1 { - t.Errorf("archive downloads = %d, want 1", n) - } - if n := cachedArtifacts(t, dir); n != 1 { - t.Errorf("cached artifacts = %d, want 1", n) - } -} - -// A denylist entry naming the package by its source repository blocks the -// mirror download of the registry identity it resolves to. -func TestMirrorDenylistSwiftSourceEntry(t *testing.T) { - upstream, downloads, _ := swiftMirrorUpstream(t) - dir := t.TempDir() - configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0") - - output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0") - - if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") { - t.Errorf("output:\n%s", output) - } - if n := downloads.Load(); n != 0 { - t.Errorf("archive downloads = %d, want 0", n) - } - if n := cachedArtifacts(t, dir); n != 0 { - t.Errorf("cached artifacts = %d, want 0", n) - } -} - -// The registry may list several identities for one repository. A source -// entry denies all of them, not only the first. -func TestMirrorDenylistSwiftSourceEntryEveryIdentity(t *testing.T) { - upstream, downloads, _ := swiftMirrorUpstream(t, "other.widget", "acme.widget") - dir := t.TempDir() - configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0") - - output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0") - - if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") { - t.Errorf("output:\n%s", output) - } - if n := downloads.Load(); n != 0 { - t.Errorf("archive downloads = %d, want 0", n) - } -} - -// A source repository entry the registry can't resolve stops the mirror -// before it downloads anything, as it stops serve: left out, it would let -// the version through. -func TestMirrorDenylistUnresolvedSwiftSource(t *testing.T) { - upstream, downloads, lookups := swiftMirrorUpstream(t) - dir := t.TempDir() - configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/unknown@1.0.0") - - output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0") - - if !strings.Contains(output, "configuring denylist") || !strings.Contains(output, "github.com/acme/unknown") { - t.Errorf("output does not report the unresolved entry:\n%s", output) - } - if strings.Contains(output, "Mirror complete") { - t.Errorf("mirror ran:\n%s", output) - } - if lookups.Load() == 0 { - t.Error("the entry was not looked up") - } - if n := downloads.Load(); n != 0 { - t.Errorf("archive downloads = %d, want 0", n) - } -} diff --git a/cmd/proxy/mirror_test.go b/cmd/proxy/mirror_test.go deleted file mode 100644 index bd39de9..0000000 --- a/cmd/proxy/mirror_test.go +++ /dev/null @@ -1,214 +0,0 @@ -package main - -import ( - "errors" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/mirror" -) - -// mirrorTestEnv carries the mirror command's arguments into the helper -// process, one per line. -const mirrorTestEnv = "PROXY_TEST_MIRROR_ARGS" - -// writeScanningMirrorConfig writes a config with scanning enabled and storage -// and database under dir. The scanner address is never reached. -func writeScanningMirrorConfig(t *testing.T, dir string) string { - t.Helper() - cfg := "storage:\n" + - " url: file://" + filepath.Join(dir, "artifacts") + "\n" + - "database:\n" + - " path: " + filepath.Join(dir, "proxy.db") + "\n" + - "scanning:\n" + - " enabled: true\n" + - " signing_key: test-signing-key\n" + - " scanners:\n" + - " - name: test\n" + - " url: http://127.0.0.1:1/scan\n" - path := filepath.Join(dir, "config.yaml") - if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil { - t.Fatalf("writing config: %v", err) - } - return path -} - -// runMirrorProcess runs proxy mirror with args in a helper process and -// checks that it exits with wantCode. -func runMirrorProcess(t *testing.T, wantCode int, args ...string) string { - t.Helper() - cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorHelperProcess$") - cmd.Env = append(os.Environ(), mirrorTestEnv+"="+strings.Join(args, "\n")) - output, err := cmd.CombinedOutput() - - code := 0 - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - code = exitErr.ExitCode() - } else if err != nil { - t.Fatalf("proxy mirror %v: %v\n%s", args, err, output) - } - if code != wantCode { - t.Fatalf("proxy mirror %v exited with status %d, want %d\n%s", args, code, wantCode, output) - } - return string(output) -} - -// With scanning enabled, proxy mirror refuses packages rather than caching -// them unscanned: scanners fetch staged bytes from the running server, which -// the command does not start. -// A refused package counts as failed, so the command exits 1. -func TestMirrorCommandRefusesPackagesWhileScanning(t *testing.T) { - dir := t.TempDir() - configPath := writeScanningMirrorConfig(t, dir) - - output := runMirrorProcess(t, 1, "-config", configPath, "pkg:npm/lodash@4.17.21") - - for _, want := range []string{mirrorScanningPackagesReason, "0 downloaded", "1 failed"} { - if !strings.Contains(output, want) { - t.Errorf("output missing %q:\n%s", want, output) - } - } - - db, err := database.Open(filepath.Join(dir, "proxy.db")) - if err != nil { - t.Fatalf("opening database: %v", err) - } - defer func() { _ = db.Close() }() - stats, err := db.GetCacheStats() - if err != nil { - t.Fatalf("GetCacheStats: %v", err) - } - if stats.TotalArtifacts != 0 { - t.Errorf("cached artifacts = %d, want 0", stats.TotalArtifacts) - } - - var stored []string - _ = filepath.WalkDir(filepath.Join(dir, "artifacts"), func(path string, d os.DirEntry, err error) error { - if err == nil && !d.IsDir() { - stored = append(stored, path) - } - return nil - }) - if len(stored) != 0 { - t.Errorf("files written to storage: %v", stored) - } -} - -// A dry run caches nothing, so it still lists what would be mirrored. -func TestMirrorCommandDryRunWhileScanning(t *testing.T) { - configPath := writeScanningMirrorConfig(t, t.TempDir()) - - output := runMirrorProcess(t, 0, "-config", configPath, "-dry-run", "pkg:npm/lodash@4.17.21") - - if !strings.Contains(output, "Would mirror 1 package versions") { - t.Errorf("dry run output:\n%s", output) - } - if strings.Contains(output, mirrorScanningPackagesReason) { - t.Errorf("dry run reported the refusal:\n%s", output) - } -} - -func TestMirrorCommandSBOMComponentVersion(t *testing.T) { - dir := t.TempDir() - configPath := writeScanningMirrorConfig(t, dir) - sbomPath := filepath.Join(dir, "bom.json") - data := []byte(`{"components":[{"name":"app","version":"1.0","purl":"pkg:docker/ghcr.io/team/app"}]}`) - if err := os.WriteFile(sbomPath, data, 0600); err != nil { - t.Fatal(err) - } - output := runMirrorProcess(t, 0, "-config", configPath, "-dry-run", "-sbom", sbomPath) - for _, want := range []string{"Would mirror 1 package versions", "pkg:docker/ghcr.io/team/app@1.0"} { - if !strings.Contains(output, want) { - t.Errorf("dry run missing %q:\n%s", want, output) - } - } -} - -func TestRefuseUnscannedMirroring(t *testing.T) { - m := mirror.New(nil, nil, nil, nil, 1) - refuseUnscannedMirroring(m) - - if m.RefusePackages != mirrorScanningPackagesReason { - t.Errorf("RefusePackages = %q, want %q", m.RefusePackages, mirrorScanningPackagesReason) - } - images, ok := m.Images.(mirror.UnavailableImages) - if !ok || images.Reason != mirrorScanningImagesReason { - t.Errorf("Images = %#v, want UnavailableImages with the scanning reason", m.Images) - } -} - -func TestMirrorHelperProcess(*testing.T) { - args := os.Getenv(mirrorTestEnv) - if args == "" { - return - } - os.Args = append([]string{"proxy", "mirror"}, strings.Split(args, "\n")...) - main() -} - -func TestMirrorExitCode(t *testing.T) { - tests := []struct { - name string - progress *mirror.Progress - want int - }{ - {"nil progress", nil, 0}, - {"everything mirrored", &mirror.Progress{Total: 2, Completed: 2}, 0}, - {"everything cached", &mirror.Progress{Total: 2, Skipped: 2}, 0}, - {"one failure", &mirror.Progress{Total: 2, Completed: 1, Failed: 1}, 1}, - {"all failed", &mirror.Progress{Total: 3, Failed: 3}, 1}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := mirrorExitCode(tt.progress); got != tt.want { - t.Errorf("mirrorExitCode() = %d, want %d", got, tt.want) - } - }) - } -} - -// proxy mirror used to exit 0 after printing "N failed", so a CI step that -// prewarms the cache passed even when every download failed (#500). An -// ecosystem the resolver rejects fails without touching the network. -func TestMirrorExitsNonZeroWhenDownloadsFail(t *testing.T) { - dir := t.TempDir() - cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorFailureProcess$") - cmd.Env = append(os.Environ(), - "PROXY_TEST_MIRROR_FAILURE=1", - "PROXY_TEST_MIRROR_DIR="+dir, - ) - output, err := cmd.CombinedOutput() - - var exitErr *exec.ExitError - if !errors.As(err, &exitErr) { - t.Fatalf("mirror with a failed download exited with %v, want status 1\n%s", err, output) - } - if code := exitErr.ExitCode(); code != 1 { - t.Fatalf("exit status = %d, want 1\n%s", code, output) - } - if !strings.Contains(string(output), "1 failed") { - t.Errorf("output does not report the failure:\n%s", output) - } - if !strings.Contains(string(output), "bogus/foo@1.0.0") { - t.Errorf("output does not list the failed package:\n%s", output) - } -} - -func TestMirrorFailureProcess(*testing.T) { - if os.Getenv("PROXY_TEST_MIRROR_FAILURE") != "1" { - return - } - dir := os.Getenv("PROXY_TEST_MIRROR_DIR") - os.Args = []string{ - "proxy", "mirror", - "--storage-url", "file://" + filepath.Join(dir, "store"), - "--database-path", filepath.Join(dir, "cache.db"), - "pkg:bogus/foo@1.0.0", - } - main() -} diff --git a/config.example.yaml b/config.example.yaml index c8ab11b..d1ed9a1 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -14,58 +14,12 @@ base_url: "http://localhost:8080" # Set to "0" to disable the timeout. Default: "30s". # http_timeout: "30s" -# Response write deadline, including download and scan time before sending data. -# Set to "0" to disable. Default: "5m". -# write_timeout: "5m" - -# Timeout for each upstream artifact download while serving requests; excludes -# scanning. Set to "0" to disable. Default: "5m". Does not limit proxy mirror. -# artifact_timeout: "5m" - -# Register DELETE /api/cache and show Delete buttons on the UI's version page, -# for removing cached artifacts so they are fetched again. Has no -# authentication: restrict /api/cache and /ui at the ingress first. -# cache_delete: false - -# Largest upstream artifact to fetch (e.g. "2GB"). Larger ones are refused -# and not cached. Default: unlimited. -# artifact_max_size: "" - -# Memory for rewritten npm and Composer metadata, so each upstream document is -# rewritten and gzip-compressed once rather than on every request. Set to "0" -# to rewrite on every request. Default: "256MB". -# metadata_rewrite_cache_size: "256MB" - # Public URL where the web UI is reached. Defaults to base_url when unset. # Set this separately when the UI is served on a different hostname than the # package endpoints — for example, the UI on a public domain behind auth while # build machines hit a Docker network alias for the package endpoints. # ui_base_url: "https://proxy.example.com/ui" -# Attribute requests to the leftmost X-Forwarded-For entry rather than the TCP -# peer address, in the structured log, the access log and the request-source -# table on /ui/analytics. -# -# Enable this only when the proxy sits behind a load balancer or ingress that -# sets the header. Any client can send it: behind one it is the only way to see -# past the hop, in front of one it lets a caller forge its own address. -# trust_forwarded_for: false - -# Show the request-source table on /ui/analytics: caller addresses, the tool -# each ran and how much each pulled. -# -# Off by default. The proxy has no authentication of its own, so leave this off -# unless /ui is gated by a reverse proxy; anyone who can reach the page can -# otherwise read the addresses of your build fleet. -# ui_request_sources: false - -# List the latest failures behind each failure count on /ui/analytics (upstream, -# storage, integrity, health probe and scan errors): the package, its upstream -# URL or storage path, and the error. URLs lose credentials and query strings. -# -# Off by default, for the same reason as ui_request_sources. -# ui_failure_details: false - # Artifact storage configuration storage: # Storage backend URL @@ -73,20 +27,9 @@ storage: # - file:///path/to/dir - Local filesystem (default) # - s3://bucket-name - Amazon S3 # - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO) - # - gs://bucket-name - Google Cloud Storage - # - azblob://container-name - Azure Blob Storage # # For S3, configure credentials via environment variables: # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION - # - # For GCS, authentication uses Application Default Credentials. On GKE with - # Workload Identity, bind the Kubernetes service account to a Google service - # account that has roles/storage.objectAdmin on the bucket. No extra config - # is needed in this file. For local development, run: - # gcloud auth application-default login - # If direct_serve is enabled, the service account also needs - # roles/iam.serviceAccountTokenCreator on itself so the IAM Credentials - # signBlob API can sign URLs without a private key. url: "" # Local filesystem path (used when url is empty) @@ -98,34 +41,8 @@ storage: # Empty or "0" means unlimited max_size: "" - # Evict artifacts that have not been served for longer than a duration - # ("30d", "72h"), independent of max_size. Resolution order: package rule, - # then ecosystem rule, then default. "0" or empty means never evict by age. - # The default only applies to ecosystems that support retention; naming an - # ecosystem or package that does not is a configuration error. See - # docs/configuration.md for the supported ecosystems. - # Generic downloads require an explicit ecosystem or package rule; - # they never inherit default, since an upstream release may disappear. - retention: - default: "" - # ecosystems: - # npm: "30d" - # generic: "30d" - # packages: - # "pkg:npm/lodash": "0" - # "pkg:generic/owner%2Frepo": "90d" - # How often to look for expired artifacts; at least "1m". - sweep_interval: "10m" - - # Store fetched artifacts. Set to false to stream every download from - # upstream without storing it; metadata filtering, cooldown and the - # denylist still apply. Useful when another cache sits in front of the - # proxy. false is incompatible with direct_serve, scanning, mirror_api and - # mirror.schedules. - cache_artifacts: true - # Redirect cached artifact downloads to presigned storage URLs (HTTP 302) - # instead of streaming through the proxy. Only effective for S3, GCS, and Azure. + # instead of streaming through the proxy. Only effective for S3 and Azure. # Leave disabled if clients reach the proxy through an authenticating gateway, # since presigned URLs bypass it. direct_serve: false @@ -153,10 +70,6 @@ database: # Example: "postgres://user:password@localhost:5432/proxy?sslmode=disable" url: "" - # How often cache hit counts and last-access times are written, batched in - # one transaction. "0" writes each hit as it happens. - hit_flush_interval: "1s" - # Logging configuration log: # Minimum log level: "debug", "info", "warn", "error" @@ -165,33 +78,16 @@ log: # Log format: "text" or "json" format: "text" -# JSONL access log. Leave path empty to disable it. -access_log: - path: "" - -# Upstream URLs for built-in routes and authentication +# Upstream registry URLs and authentication upstream: - # Hosts allowed to resolve to private, ULA, or CGNAT addresses - allow_private_hosts: [] - - # Permit upstream requests and redirects to loopback addresses - allow_loopback: false - # npm registry URL npm: "https://registry.npmjs.org" - # Always request full npm packuments so served metadata carries publish - # times ("time" map) even when cooldown is disabled. Needed by clients that - # gate on publish age, e.g. Yarn's npmMinimalAgeGate. Default: false. - # npm_full_metadata: true - # npm packages served only from a private registry, by name pattern. - # Matching packages are never looked up on the npm registry above. - # npm_routes: - # "@example/*": "https://npm.example.com/registry" + # Maven repository URL (used by /maven endpoint) + maven: "https://repo1.maven.org/maven2" - # Follow the npm registry's change feed, so a new release is fetched on the - # next request instead of after metadata_ttl. Requires cache_metadata. - # npm_change_feed: true + # Gradle Plugin Portal Maven URL (fallback for plugin marker artifacts) + gradle_plugin_portal: "https://plugins.gradle.org/m2" # Cargo sparse index URL cargo: "https://index.crates.io" @@ -199,187 +95,14 @@ upstream: # Cargo crate download URL cargo_download: "https://static.crates.io/crates" - # RubyGems registry URL - gem: "https://rubygems.org" - - # Go module proxy URL - go: "https://proxy.golang.org" - - # Hex repository URL - hex: "https://repo.hex.pm" - - # Hex API URL used for package timestamps - hex_api: "https://hex.pm" - - # pub registry URL - pub: "https://pub.dev" - - # PyPI index and API URL - pypi: "https://pypi.org" - - # PyPI package download URL - pypi_download: "https://files.pythonhosted.org" - - # Maven repository URL (used by /maven endpoint) - maven: "https://repo1.maven.org/maven2" - - # Additional Maven repositories, served at /maven/{name}/. A name must not - # be the start of a group ID (such as "org" or "tools"), since it would hide - # those groups in the default repository. - # maven_repositories: - # grails: "https://repo.grails.org/grails/restricted" - - # Gradle Plugin Portal Maven URL (fallback for plugin marker artifacts) - gradle_plugin_portal: "https://plugins.gradle.org/m2" - - # NuGet API URL - nuget: "https://api.nuget.org" - - # NuGet search API URL - nuget_search: "https://azuresearch-usnc.nuget.org" - - # Packagist API URL - composer: "https://packagist.org" - - # Packagist repository URL - composer_repository: "https://repo.packagist.org" - - # Composer packages served only from a private repository, by name pattern. - # Matching packages are never looked up on composer_repository. - # composer_routes: - # "example/*": "https://composer.example.com/packages" - - # Follow the change feed of the Packagist at composer, so a new release is - # fetched on the next request instead of after metadata_ttl. Requires - # cache_metadata. - # composer_change_feed: true - - # Conan registry URL - conan: "https://center.conan.io" - - # Chef Supermarket URL (used by /chef/). Point it at a private - # Supermarket to proxy that instead. - chef: "https://supermarket.chef.io" - - # Conda channel base URL - conda: "https://conda.anaconda.org" - - # CRAN mirror URL - cran: "https://cloud.r-project.org" - - # Julia package server URL - julia: "https://pkg.julialang.org" - - # Swift Package Registry URL (used by /swift endpoint) - swift: "https://tuist.dev/api/registry/swift" - - # Default OCI registry URL for unprefixed /v2 requests - oci_default: "https://registry-1.docker.io" - - # Debian/APT repository URL (used by /debian endpoint) - debian: "http://deb.debian.org/debian" - - # Additional Debian/APT archives, served at /debian/{name}/. - # The names "pool" and "dists" are reserved for the main archive's own paths. - debian_repositories: - security: "https://security.debian.org/debian-security" - - # RPM repository URL (used by /rpm endpoint) - rpm: "https://dl.fedoraproject.org/pub/fedora/linux" - - # Homebrew JSON API URL (used by /homebrew endpoint) - homebrew_api: "https://formulae.brew.sh/api" - - # Homebrew artifact registry URL (used for /v2/homebrew/core requests) - homebrew_artifact: "https://ghcr.io" - - # Named HTTP Helm chart repositories (used by /helm/{name}/) - # helm: - # bitnami: "https://charts.bitnami.com/bitnami" - - # Named OCI registries. Use the upstream/{name}/ repository prefix, e.g. - # oci://proxy.example.com/upstream/ghcr/owner/chart. - # oci: - # ghcr: "https://ghcr.io" - - # Registries a named OCI upstream mirrors, e.g. when ghcr above points at - # an Artifactory remote of ghcr.io. containerd mirror requests whose ns - # parameter names one of these hosts are served by that upstream. - # oci_mirrors: - # ghcr: ["ghcr.io"] - - # Named Alpine APK repositories (used by /apk/{name}/). - # Defaults to {"alpine": "https://dl-cdn.alpinelinux.org/alpine"} when empty; - # configuring any entry replaces that default. - # apk: - # alpine: "https://dl-cdn.alpinelinux.org/alpine" - # private: "https://apk.example.com" - - # Named Arch Linux pacman mirrors (used by /arch/{name}/). - # Defaults to {"archlinux": "https://geo.mirror.pkgbuild.com"} when empty; - # configuring any entry replaces that default. Arch Linux ARM needs a - # regional HTTPS mirror of your choice, not the mirror.archlinuxarm.org - # redirector. - # arch: - # archlinux: "https://geo.mirror.pkgbuild.com" - # archlinuxarm: "https://.mirror.archlinuxarm.org" - - # Named FreeBSD pkg mirrors (used by /freebsd/{name}/), for ports and - # pkgbase repositories alike. - # Defaults to {"freebsd": "https://pkg.freebsd.org"} when empty; - # configuring any entry replaces that default. - # freebsd: - # freebsd: "https://pkg.freebsd.org" - # poudriere: "https://packages.example.com" - - # Named generic HTTP upstreams (used by /generic/{name}/). The remaining - # request path and query are appended to the upstream URL. GitHub release - # assets ({owner}/{repo}/releases/download/{tag}/{asset}) are cached - # immutably; other paths use the metadata cache with stale-on-error. - # generic: - # github: "https://github.com" - # github-api: "https://api.github.com" - # apache: "https://archive.apache.org" - - # Paths a generic upstream may serve, as path.Match patterns for the path - # after /generic/{name}/; a pattern ending in /** also matches everything - # nested below its prefix. Other paths answer 404. Required for an upstream - # that upstream.auth has credentials for, such as a GitHub token for - # api.github.com; use a fine-grained token with read-only access to public - # repositories. - # generic_paths: - # github-api: - # - "repos/*/*/releases" - # - "repos/*/*/releases/**" - # - "repos/*/*/tags" - - # Version-pinned paths of generic upstreams, cached immutably like GitHub - # release assets. Each pattern must match the whole path after - # /generic/{name}/ and have `version` and `file` named groups; an optional - # `name` group names the package. - # generic_artifacts: - # apache: - # - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' - - # Named Git servers (used by /git/{name}/{repo}), served read-only from - # cached bare repositories. repos limits which repositories may be served - # and is required when upstream.auth has credentials for the server. - # git: - # github: "https://github.com" - # wikimedia: - # url: "https://gerrit.wikimedia.org/r" - # repos: ["mediawiki/**"] - # Authentication for upstream registries - # Keys are absolute URL scopes. Scheme, host, effective port, and path - # segment boundaries must match; the longest matching scope wins. + # Keys are URL prefixes matched against request URLs. # Values can reference environment variables using ${VAR_NAME} syntax. # # Supported auth types: # - bearer: Authorization header with Bearer token # - basic: Authorization header with Basic auth (username:password) # - header: Custom header name and value - # - ecr: AWS ECR auto-refreshing token via the AWS SDK credential chain auth: # Example: npm with bearer token # "https://registry.npmjs.org": @@ -403,27 +126,11 @@ upstream: # header_name: "X-Auth-Token" # header_value: "${MAVEN_TOKEN}" - # Example: private AWS ECR registry (12h tokens auto-refreshed via - # ecr:GetAuthorizationToken; credentials come from the AWS SDK default - # chain, so IRSA / instance profiles / AWS_* env vars all work; the region - # is inferred from the private ECR hostname) - # "https://123456789012.dkr.ecr.eu-west-1.amazonaws.com": - # type: ecr - -# Git repository cache, used when upstream.git is set -# git: -# cache_dir: ./cache/git # local disk, used by this process only -# max_size: "" # evict least recently used repositories past this -# max_repo_size: 5GB # refuse a repository that grows past this -# refresh_wait: 5s # then serve the cached copy if the upstream check is slow -# stale_max_age: 24h # never serve a copy not checked for this long - # Gradle HttpBuildCache configuration gradle: build_cache: - # PUT uploads are rejected by default; set to false on the instance - # your CI jobs upload to - read_only: true + # Set to true to disable PUT uploads (read-only cache mode) + read_only: false # Maximum accepted Gradle cache upload body size # Required and must be > 0 @@ -439,35 +146,6 @@ gradle: # How often eviction runs when max_age or max_size is set sweep_interval: "10m" -# Pushing package archives to /api/hosted (disabled by default). -# A pushed version replaces the upstream one for every client, so a token is -# required to push and delete. -# hosted: -# enabled: true -# token: "${PROXY_HOSTED_TOKEN}" -# max_upload_size: "100MB" -# # Namespaces served from pushed packages only; the upstream is never asked -# # about them, so nobody can mix in versions by publishing the same name. -# npm_scopes: ["@acme"] -# composer_vendors: ["acme"] - -# Project inventory (disabled by default). Repositories upload an SBOM to -# PUT /api/projects/{project}/sbom, and the UI shows which project uses -# which package versions. Project names are visible on the unauthenticated UI. -# projects: -# enabled: true -# token: "${PROXY_PROJECTS_TOKEN}" -# # Drop projects that have not uploaded for this long ("" keeps them) -# max_age: "90d" - -# Re-mirror container image tags on a schedule, so a moved tag is picked up -# even when nobody pulls it. An unchanged tag costs one HEAD request. -# mirror: -# schedules: -# - images: ["pkg:docker/library/alpine@3.20", "pkg:docker/library/nginx@stable"] -# interval: "6h" -# platforms: ["linux/amd64"] - # Health endpoint configuration. health: # Minimum time between storage backend probes. @@ -482,13 +160,6 @@ health: # to spot malicious releases before they're pulled into projects. # Supported durations: "7d" (days), "48h" (hours), "30m" (minutes), "0" (disabled) cooldown: - # Independent policies selected through /cooldown/{name}/{registry}/. - # profiles: - # 3d: - # default: "3d" - # 7d: - # default: "7d" - # Global default cooldown for all ecosystems # default: "3d" @@ -497,51 +168,7 @@ cooldown: # npm: "7d" # cargo: "0" - # Per-package overrides (keyed by PURL). Keys are normalized, so npm scopes - # may use either @scope or the canonical %40scope form. + # Per-package overrides (keyed by PURL) # packages: # "pkg:npm/lodash": "0" # "pkg:npm/@babel/core": "14d" - - # Per-package glob overrides, after exact packages and before ecosystems. - # package_patterns: - # "pkg:npm/@example/*": "0" - -# Exact versions to deny, independently of cooldown and scanning. -# Metadata filtering: npm, PyPI and Cargo. Shared artifact downloads, including -# cache hits, are blocked with 403; signed APT metadata is left unchanged. -# Use versioned PURLs without qualifiers/subpaths. Restart after changes. -# denylist: -# packages: -# - "pkg:pypi/requests@2.31.0" -# - "pkg:cargo/some-crate@1.2.3" -# - "pkg:npm/%40scope/example@4.5.6" - -# Pre-cache artifact scanning. When enabled, every artifact is staged into -# storage and scanned by the configured scanners before it is committed to -# the cache and served to clients. Scanners never receive artifact bytes -# directly — each notify call includes a short-lived signed URL that the -# scanner fetches itself, so the proxy stays agnostic to trivy/ClamAV/Wiz/ -# any custom service. Scanners run concurrently; the first "block" verdict -# wins and cancels the rest. -# scanning: -# enabled: true -# fail_open: false -# timeout: 30s -# -# # Authenticates pull requests to the internal scan-fetch route. -# # Required whenever enabled is true. Supports ${VAR_NAME} expansion. -# signing_key: ${PROXY_SCANNING_SIGNING_KEY} -# -# # Address scanners use to reach this proxy to pull staged artifacts. -# # Defaults to base_url. -# # fetch_base_url: http://proxy.internal:8080 -# -# scanners: -# - name: clamav -# url: http://clamav-adapter:8080/scan -# mode: block -# - name: trivy -# url: http://trivy-adapter:8081/scan -# mode: monitor -# ecosystems: [npm, pypi] diff --git a/deploy/charts/proxy/.helmignore b/deploy/charts/proxy/.helmignore deleted file mode 100644 index 4f7d63c..0000000 --- a/deploy/charts/proxy/.helmignore +++ /dev/null @@ -1,6 +0,0 @@ -.DS_Store -.git/ -.github/ -*.swp -*.tmp -*.tgz diff --git a/deploy/charts/proxy/Chart.yaml b/deploy/charts/proxy/Chart.yaml deleted file mode 100644 index 4e6bff7..0000000 --- a/deploy/charts/proxy/Chart.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v2 -name: proxy -description: A caching proxy for package registries -type: application -version: 0.0.0 -appVersion: "0.0.0" -home: https://github.com/git-pkgs/proxy -sources: - - https://github.com/git-pkgs/proxy -annotations: - artifacthub.io/license: MIT diff --git a/deploy/charts/proxy/templates/NOTES.txt b/deploy/charts/proxy/templates/NOTES.txt deleted file mode 100644 index 3b29f51..0000000 --- a/deploy/charts/proxy/templates/NOTES.txt +++ /dev/null @@ -1,15 +0,0 @@ -git-pkgs proxy has been installed. - -The default base URL is intended for local port forwarding. Before exposing the -proxy, set config.data.base_url to the URL used by package-manager clients. - -To access the proxy locally: - - kubectl -n {{ .Release.Namespace }} port-forward service/{{ include "proxy.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} - -Then visit http://localhost:{{ .Values.service.port }}/. - -{{- if not .Values.persistence.enabled }} -WARNING: persistence is disabled. Cached artifacts and the default SQLite -database will be lost when the pod is replaced. -{{- end }} diff --git a/deploy/charts/proxy/templates/_helpers.tpl b/deploy/charts/proxy/templates/_helpers.tpl deleted file mode 100644 index 9408a9f..0000000 --- a/deploy/charts/proxy/templates/_helpers.tpl +++ /dev/null @@ -1,46 +0,0 @@ -{{/* Expand the chart name. */}} -{{- define "proxy.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* Create a release-specific, DNS-safe resource name. */}} -{{- define "proxy.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := include "proxy.name" . }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{- define "proxy.labels" -}} -helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{ include "proxy.selectorLabels" . }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{- define "proxy.selectorLabels" -}} -app.kubernetes.io/name: {{ include "proxy.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{- define "proxy.configMapName" -}} -{{- default (include "proxy.fullname" .) .Values.config.existingConfigMap }} -{{- end }} - -{{- define "proxy.claimName" -}} -{{- default (include "proxy.fullname" .) .Values.persistence.existingClaim }} -{{- end }} - -{{- define "proxy.image" -}} -{{- if .Values.image.digest -}} -{{- printf "%s@%s" .Values.image.repository .Values.image.digest -}} -{{- else -}} -{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) -}} -{{- end -}} -{{- end }} diff --git a/deploy/charts/proxy/templates/configmap.yaml b/deploy/charts/proxy/templates/configmap.yaml deleted file mode 100644 index 40aad7b..0000000 --- a/deploy/charts/proxy/templates/configmap.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if not .Values.config.existingConfigMap }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "proxy.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "proxy.labels" . | nindent 4 }} -data: - {{ required "config.existingConfigMapKey is required" .Values.config.existingConfigMapKey }}: | - {{- toYaml .Values.config.data | nindent 4 }} -{{- end }} diff --git a/deploy/charts/proxy/templates/deployment.yaml b/deploy/charts/proxy/templates/deployment.yaml deleted file mode 100644 index f04728a..0000000 --- a/deploy/charts/proxy/templates/deployment.yaml +++ /dev/null @@ -1,121 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "proxy.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "proxy.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - strategy: - {{- toYaml .Values.deploymentStrategy | nindent 4 }} - selector: - matchLabels: - {{- include "proxy.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "proxy.selectorLabels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - annotations: - checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} - {{- with .Values.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - automountServiceAccountToken: false - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.extraInitContainers }} - initContainers: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: {{ .Chart.Name }} - image: {{ include "proxy.image" . | quote }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - securityContext: - {{- toYaml .Values.containerSecurityContext | nindent 12 }} - args: - - serve - - -config - - /etc/proxy/{{ .Values.config.existingConfigMapKey }} - {{- with .Values.extraEnv }} - env: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.extraEnvFrom }} - envFrom: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: {{ .Values.service.containerPort }} - protocol: TCP - startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} - readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} - livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: config - mountPath: /etc/proxy/{{ .Values.config.existingConfigMapKey }} - subPath: {{ .Values.config.existingConfigMapKey }} - readOnly: true - - name: data - mountPath: {{ .Values.persistence.mountPath }} - - name: tmp - mountPath: /tmp - {{- with .Values.extraVolumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - volumes: - - name: config - configMap: - name: {{ include "proxy.configMapName" . }} - items: - - key: {{ required "config.existingConfigMapKey is required" .Values.config.existingConfigMapKey }} - path: {{ .Values.config.existingConfigMapKey }} - - name: data - {{- if .Values.persistence.enabled }} - persistentVolumeClaim: - claimName: {{ include "proxy.claimName" . }} - {{- else }} - emptyDir: {} - {{- end }} - - name: tmp - emptyDir: - {{- with .Values.tmpDir.sizeLimit }} - sizeLimit: {{ . }} - {{- else }} - {} - {{- end }} - {{- with .Values.extraVolumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.topologySpreadConstraints }} - topologySpreadConstraints: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deploy/charts/proxy/templates/ingress.yaml b/deploy/charts/proxy/templates/ingress.yaml deleted file mode 100644 index cc90155..0000000 --- a/deploy/charts/proxy/templates/ingress.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- if .Values.ingress.enabled }} -{{- if not .Values.ingress.hosts }} -{{- fail "ingress.hosts must be set when ingress.enabled=true" }} -{{- end }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ include "proxy.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "proxy.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- with .Values.ingress.className }} - ingressClassName: {{ . | quote }} - {{- end }} - {{- with .Values.ingress.tls }} - tls: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path | quote }} - pathType: {{ .pathType }} - backend: - service: - name: {{ include "proxy.fullname" $ }} - port: - number: {{ $.Values.service.port }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deploy/charts/proxy/templates/pvc.yaml b/deploy/charts/proxy/templates/pvc.yaml deleted file mode 100644 index 80687b2..0000000 --- a/deploy/charts/proxy/templates/pvc.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "proxy.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "proxy.labels" . | nindent 4 }} - {{- with .Values.persistence.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - accessModes: - {{- toYaml .Values.persistence.accessModes | nindent 4 }} - {{- with .Values.persistence.storageClass }} - storageClassName: {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.persistence.size }} -{{- end }} diff --git a/deploy/charts/proxy/templates/service.yaml b/deploy/charts/proxy/templates/service.yaml deleted file mode 100644 index ee5931e..0000000 --- a/deploy/charts/proxy/templates/service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "proxy.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "proxy.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - name: http - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - selector: - {{- include "proxy.selectorLabels" . | nindent 4 }} diff --git a/deploy/charts/proxy/values.yaml b/deploy/charts/proxy/values.yaml deleted file mode 100644 index bbb253f..0000000 --- a/deploy/charts/proxy/values.yaml +++ /dev/null @@ -1,173 +0,0 @@ -# More than one replica requires config.data.database on Postgres and -# deploymentStrategy: RollingUpdate. Storage can be object storage (with -# persistence disabled) or a file:// URL on a ReadWriteMany volume -# (persistence.accessModes: [ReadWriteMany]). With upstream.git, each -# replica also needs its own git.cache_dir. -replicaCount: 1 - -nameOverride: "" -fullnameOverride: "" - -image: - repository: ghcr.io/git-pkgs/proxy - # An empty tag uses the chart appVersion. - tag: "" - # When set, digest takes precedence over tag. - digest: "" - pullPolicy: IfNotPresent - -imagePullSecrets: [] - -service: - type: ClusterIP - port: 8080 - # Keep this aligned with config.data.listen (or the listen address in an - # existing ConfigMap). - containerPort: 8080 - -# The generated configuration is ignored when existingConfigMap is set. -config: - existingConfigMap: "" - existingConfigMapKey: config.yaml - data: - listen: ":8080" - # Set this to the URL package-manager clients use to reach the proxy. - base_url: "http://localhost:8080" - storage: - url: "file:///data/artifacts" - database: - driver: sqlite - path: "/data/proxy.db" - log: - level: info - format: json - -# Environment variables override values from the configuration file. This is -# also the recommended way to supply secret values such as database passwords -# and object-storage credentials. -extraEnv: [] -# - name: PROXY_DATABASE_URL -# valueFrom: -# secretKeyRef: -# name: proxy-database -# key: url - -extraEnvFrom: [] -# - secretRef: -# name: proxy-object-storage - -# Additional volumes for the pod and mounts for the proxy container, for -# example a ConfigMap holding a private CA certificate. Go adds certificates -# from the directories in SSL_CERT_DIR to its trusted roots (set it with -# extraEnv); SSL_CERT_FILE replaces the system bundle instead. All upstream -# TLS, including Git fetches, goes through the proxy process. -extraVolumes: [] -# - name: corp-ca -# configMap: -# name: corp-ca - -extraVolumeMounts: [] -# - name: corp-ca -# mountPath: /etc/proxy/ca -# readOnly: true - -# Init containers run before the proxy starts. They can use extraVolumes, but -# extraVolumeMounts applies only to the proxy container, so each init container -# needs its own volumeMounts. -extraInitContainers: [] -# - name: wait-for-db -# image: busybox:1.37 -# command: ["sh", "-c", "until nc -z postgres 5432; do sleep 2; done"] - -# With SQLite, the database file needs a local disk or block-backed volume, -# even with one replica. ReadWriteOnce doesn't guarantee that: pick a -# storageClass that isn't NFS or another network filesystem. If only network -# filesystem storage is available, set config.data.database to Postgres. -# Artifact storage can stay on NFS on a separate volume (extraVolumes and -# extraVolumeMounts, with config.data.storage.url pointing at it). -persistence: - enabled: true - # Keep this aligned with config.data.storage.url and config.data.database.path - # (or the equivalent paths in an existing ConfigMap). - mountPath: /data - existingClaim: "" - annotations: {} - accessModes: - - ReadWriteOnce - size: 10Gi - storageClass: "" - -deploymentStrategy: - type: Recreate - -ingress: - enabled: false - className: "" - annotations: {} - hosts: [] - # - host: proxy.example.com - # paths: - # - path: / - # pathType: Prefix - tls: [] - # - secretName: proxy-tls - # hosts: - # - proxy.example.com - -podAnnotations: {} -podLabels: {} - -podSecurityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - -containerSecurityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - -# /tmp is an emptyDir, since the root filesystem is read-only. -tmpDir: - sizeLimit: "" - -resources: {} -# requests: -# cpu: 100m -# memory: 128Mi -# limits: -# memory: 512Mi - -startupProbe: - tcpSocket: - port: http - failureThreshold: 30 - periodSeconds: 2 - -# /health checks both the database and storage backends and can take up to ten -# seconds. It is intentionally a readiness check rather than a liveness check. -readinessProbe: - httpGet: - path: /health - port: http - timeoutSeconds: 11 - periodSeconds: 15 - failureThreshold: 2 - -livenessProbe: - tcpSocket: - port: http - periodSeconds: 20 - failureThreshold: 3 - -terminationGracePeriodSeconds: 30 - -nodeSelector: {} -tolerations: [] -affinity: {} -topologySpreadConstraints: [] diff --git a/deploy/grafana/git-pkgs-proxy.json b/deploy/grafana/git-pkgs-proxy.json deleted file mode 100644 index 2592625..0000000 --- a/deploy/grafana/git-pkgs-proxy.json +++ /dev/null @@ -1,2935 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Download volume, cache composition and upstream health for the git-pkgs proxy. Accumulated download size is cache hits multiplied by artifact size, in total and per ecosystem.", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Overview", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Total bytes served to clients from cache: for each artifact, its size multiplied by the number of times it was served. This is traffic carried, not storage used, so it is normally far larger than the cache on disk. Artifacts evicted from the cache stop contributing.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": true, - "instant": false, - "refId": "A" - } - ], - "title": "Accumulated download size", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Bytes the cached artifacts currently occupy in storage \u2014 what the cache costs to keep. Not an average, and not the same as accumulated download size, which is what the cache has served.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 5, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(proxy_cache_size_bytes{job=~\"$job\"})", - "range": true, - "instant": false, - "refId": "A" - } - ], - "title": "Cache size on disk", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Accumulated download size divided by the bytes currently cached: how many times over the cache has served what it stores. This is the number that makes the two size figures beside it comparable, and it is the clearest single measure of what the cache is worth.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "suffix:\u00d7" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 10, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"}) / sum(proxy_ecosystem_cache_size_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": true, - "instant": false, - "refId": "A" - } - ], - "title": "Served per byte cached", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Number of distinct artifacts held in the cache. A small count with a large accumulated download size just means a few artifacts are being served repeatedly, which is the cache doing its job.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 15, - "y": 1 - }, - "id": 5, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(proxy_cached_artifacts_total{job=~\"$job\"})", - "range": true, - "instant": false, - "refId": "A" - } - ], - "title": "Cached artifacts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Share of artifact lookups served from cache over the dashboard's time range. Reads as No data when nothing was requested in that range \u2014 a range with no traffic has no hit ratio, which is not the same as a hit ratio of zero.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds", - "fixedColor": "text" - }, - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 0.5 - }, - { - "color": "green", - "value": 0.8 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\", kind=\"artifact\"}[$__range]))\n/\n(\n sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\", kind=\"artifact\"}[$__range]))\n +\n sum(increase(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\", kind=\"artifact\"}[$__range]))\n)", - "range": true, - "instant": false, - "refId": "A" - } - ], - "title": "Artifact cache hit ratio", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 7, - "panels": [], - "title": "Download volume", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Accumulated bytes served from cache, per ecosystem, largest first. One measure across categories, so a single hue carries magnitude.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "blue" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 8, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [], - "fields": "/^Value$/", - "values": true - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "text", - "legend": { - "showLegend": false - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": false, - "instant": true, - "refId": "A", - "legendFormat": "{{ecosystem}}", - "format": "table" - } - ], - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "indexByName": {}, - "renameByName": {} - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Value" - } - ] - } - } - ], - "title": "Download size by ecosystem", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "How the accumulated total has grown, stacked by ecosystem. A step down means artifacts were evicted, which removes their historical hits from the total.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 18, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 9, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{ecosystem}}" - } - ], - "title": "Accumulated download size over time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Every per-ecosystem figure the charts leave to a tooltip.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Downloaded" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge", - "valueDisplayMode": "text" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache size" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 10, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": true - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Downloaded" - } - ] - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": false, - "instant": true, - "refId": "A", - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_artifact_downloads{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": false, - "instant": true, - "refId": "B", - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_cache_size_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": false, - "instant": true, - "refId": "C", - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_cached_artifacts{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": false, - "instant": true, - "refId": "D", - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_packages{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": false, - "instant": true, - "refId": "E", - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_versions{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": false, - "instant": true, - "refId": "F", - "format": "table" - } - ], - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "ecosystem", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true - }, - "indexByName": {}, - "renameByName": { - "ecosystem": "Ecosystem", - "Value #A": "Downloaded", - "Value #B": "Downloads", - "Value #C": "Cache size", - "Value #D": "Artifacts", - "Value #E": "Packages", - "Value #F": "Versions" - } - } - } - ], - "title": "Per-ecosystem breakdown", - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 11, - "panels": [], - "title": "Traffic and cache", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Responses per second, by package ecosystem. The ecosystem label on this metric is derived from the request path, so it uses route names (rubygems, packagist, debian, other) rather than the package-ecosystem names the cache metrics use. It is filtered by the Route variable, not Ecosystem.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (rate(proxy_requests_total{job=~\"$job\", ecosystem=~\"$route\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{ecosystem}}" - } - ], - "title": "Request rate by route", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Artifact lookups per second that were served from cache versus fetched upstream.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 28 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (kind) (rate(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{kind}} hits" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (kind) (rate(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "B", - "legendFormat": "{{kind}} misses" - } - ], - "title": "Cache hits and misses", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "95th percentile time to serve a proxy request, by ecosystem. The ecosystem label on this metric is derived from the request path, so it uses route names (rubygems, packagist, debian, other) rather than the package-ecosystem names the cache metrics use. It is filtered by the Route variable, not Ecosystem.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 36 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le, ecosystem) (rate(proxy_request_duration_seconds_bucket{job=~\"$job\", ecosystem=~\"$route\"}[$__rate_interval])))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{ecosystem}}" - } - ], - "title": "Request duration p95", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "95th percentile time spent fetching an artifact from the upstream registry, by ecosystem. Only cache misses reach an upstream. The ecosystem label on this metric is the handler's own name, which differs from both the cache metrics (composer vs packagist, gem vs rubygems, go vs golang) and the route names, so it is filtered by the Upstream variable.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 36 - }, - "id": 15, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le, ecosystem) (rate(proxy_upstream_fetch_duration_seconds_bucket{job=~\"$job\", ecosystem=~\"$upstream\"}[$__rate_interval])))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{ecosystem}}" - } - ], - "title": "Upstream fetch duration p95", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 44 - }, - "id": 16, - "panels": [], - "title": "Reliability", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Upstreams whose artifact-fetch breaker is open. While a breaker is open, cache misses for that host return 502 without contacting the upstream. Alert on this being above zero for more than a few minutes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds", - "fixedColor": "text" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 0, - "y": 45 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "count(proxy_circuit_breaker_state{job=~\"$job\"} == 2) or vector(0)", - "range": true, - "instant": false, - "refId": "A" - } - ], - "title": "Circuit breakers open", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Requests currently in flight.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 4, - "y": 45 - }, - "id": 18, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(proxy_active_requests{job=~\"$job\"})", - "range": true, - "instant": false, - "refId": "A" - } - ], - "title": "Active requests", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Circuit breaker trips per upstream registry. A breaker is created per host the proxy fetches artifacts from, and only reports once it has tripped at least once, so an empty panel means no upstream has failed repeatedly.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 8, - "y": 45 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (registry) (increase(proxy_circuit_breaker_trips_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "{{registry}}", - "range": true, - "instant": false, - "refId": "A" - } - ], - "title": "Circuit breaker trips", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Upstream fetch failures per second, by ecosystem and error type. The ecosystem label on this metric is the handler's own name, which differs from both the cache metrics (composer vs packagist, gem vs rubygems, go vs golang) and the route names, so it is filtered by the Upstream variable.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 45 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem, error_type) (rate(proxy_upstream_errors_total{job=~\"$job\", ecosystem=~\"$upstream\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{ecosystem}} \u00b7 {{error_type}}" - } - ], - "title": "Upstream errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Storage read/write failures, cached artifacts that failed hash verification on read, and storage health probe failures.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 45 - }, - "id": 21, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (operation) (rate(proxy_storage_errors_total{job=~\"$job\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "storage \u00b7 {{operation}}" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (rate(proxy_integrity_failures_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "B", - "legendFormat": "integrity \u00b7 {{ecosystem}}" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (step) (rate(proxy_health_probe_failures_total{job=~\"$job\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "C", - "legendFormat": "health probe \u00b7 {{step}}" - } - ], - "title": "Storage and integrity failures", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 52 - }, - "id": 22, - "panels": [], - "title": "Storage and scanning", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "95th percentile storage read/write latency, by operation.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 23, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(proxy_storage_operation_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{operation}}" - } - ], - "title": "Storage operation latency p95", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Storage operations per second, by operation.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 53 - }, - "id": 24, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (operation) (rate(proxy_storage_operation_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{operation}}" - } - ], - "title": "Storage operation rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Completed pre-cache scans per second, by scanner. Empty when scanning is not configured.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 61 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (scanner) (rate(proxy_scan_duration_seconds_count{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{scanner}}" - } - ], - "title": "Scan rate by scanner", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "95th percentile pre-cache scan duration, by scanner. A block-mode scanner's latency is on the critical path of a cache miss.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 61 - }, - "id": 26, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le, scanner) (rate(proxy_scan_duration_seconds_bucket{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval])))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{scanner}}" - } - ], - "title": "Scan duration p95", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Artifacts a block-mode scanner refused, and scan calls that failed, timed out or were cancelled. A rising error rate means artifacts are being admitted without a verdict.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 61 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem, scanner) (rate(proxy_scan_blocked_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "blocked \u00b7 {{ecosystem}} \u00b7 {{scanner}}" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (scanner, error_type) (rate(proxy_scan_errors_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "B", - "legendFormat": "error \u00b7 {{scanner}} \u00b7 {{error_type}}" - } - ], - "title": "Artifacts blocked and scan errors", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 69 - }, - "id": 28, - "panels": [], - "title": "Cache composition", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Bytes held in the cache per ecosystem. A drop is an eviction sweep.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 70 - }, - "id": 29, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_cache_size_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{ecosystem}}" - } - ], - "title": "Cache size by ecosystem", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Artifacts held in the cache per ecosystem.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 70 - }, - "id": 30, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_cached_artifacts{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{ecosystem}}" - } - ], - "title": "Cached artifacts by ecosystem", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Package and version rows known per ecosystem. These grow as metadata is fetched, independently of what is cached.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 78 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_packages{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "packages \u00b7 {{ecosystem}}" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (proxy_ecosystem_versions{job=~\"$job\", ecosystem=~\"$ecosystem\"})", - "range": true, - "instant": false, - "refId": "B", - "legendFormat": "versions \u00b7 {{ecosystem}}" - } - ], - "title": "Known packages and versions", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 86 - }, - "id": 32, - "panels": [], - "title": "Request sources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Response body bytes per second written to clients. The ecosystem label on this counter comes from the request path, not the package record, so it is filtered by the Route variable and reports route names (debian, and other for the UI and health checks). Distinct from proxy_ecosystem_downloaded_bytes, which is derived from the database as cache hits times artifact size.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 18, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 87 - }, - "id": 33, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (ecosystem) (rate(proxy_response_bytes_total{job=~\"$job\", ecosystem=~\"$route\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{ecosystem}}" - } - ], - "title": "Bytes served by route", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Response bytes per second by client tool, identified from the User-Agent. The label set is closed \u2014 an unrecognised User-Agent reports as 'other' \u2014 so a caller cannot create new time series. Caller addresses are deliberately not exported as labels; see the access log or the proxy's own analytics page for per-address detail.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 18, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 87 - }, - "id": 34, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (client) (rate(proxy_client_response_bytes_total{job=~\"$job\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{client}}" - } - ], - "title": "Bytes served by client", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Requests per second by client tool. Useful for spotting a runaway CI job or a tool that is not honouring caches.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 96 - }, - "id": 35, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (client) (rate(proxy_client_requests_total{job=~\"$job\"}[$__rate_interval]))", - "range": true, - "instant": false, - "refId": "A", - "legendFormat": "{{client}}" - } - ], - "title": "Requests by client", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "1m", - "schemaVersion": 39, - "tags": [ - "git-pkgs", - "proxy", - "cache" - ], - "templating": { - "list": [ - { - "current": {}, - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(proxy_cache_size_bytes, job)", - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(proxy_cache_size_bytes, job)", - "refId": "job" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(proxy_ecosystem_downloaded_bytes, ecosystem)", - "hide": 0, - "includeAll": true, - "label": "Ecosystem", - "multi": true, - "name": "ecosystem", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(proxy_ecosystem_downloaded_bytes, ecosystem)", - "refId": "ecosystem" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(proxy_requests_total, ecosystem)", - "description": "Route names taken from the request path. These differ from the package-ecosystem names used by the cache metrics.", - "hide": 0, - "includeAll": true, - "label": "Route", - "multi": true, - "name": "route", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(proxy_requests_total, ecosystem)", - "refId": "route" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(proxy_upstream_fetch_duration_seconds_count, ecosystem)", - "description": "Ecosystem names as the upstream fetch metrics report them, taken from the handler rather than the package record: composer, gem and go where the cache metrics say packagist, rubygems and golang.", - "hide": 0, - "includeAll": true, - "label": "Upstream", - "multi": true, - "name": "upstream", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(proxy_upstream_fetch_duration_seconds_count, ecosystem)", - "refId": "upstream" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "git-pkgs proxy", - "uid": "git-pkgs-proxy", - "version": 1, - "weekStart": "" -} diff --git a/docs/architecture.md b/docs/architecture.md index 1d17426..f04d548 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -34,7 +34,7 @@ The proxy is a caching HTTP server that sits between package manager clients and 1. Client requests `GET /npm/lodash` 2. NPMHandler receives request 3. Handler fetches metadata from upstream `registry.npmjs.org/lodash` -4. Handler rewrites tarball URLs in metadata to point at proxy, and filters versions withheld by cooldown or the denylist (`internal/metadata/npm`) +4. Handler rewrites tarball URLs in metadata to point at proxy 5. Handler returns modified metadata to client Metadata is not cached - always fetched fresh. This ensures clients see new versions immediately. @@ -51,29 +51,16 @@ Metadata is not cached - always fetched fresh. This ensures clients see new vers - Open file from storage - Record hit (increment counter, update last_accessed_at) - Return reader to handler - - Handler streams file to client, checking it against the recorded - SHA-256 `content_hash` (the registry's own integrity was checked when the - file was stored); a mismatch aborts the response and clears the record + - Handler streams file to client **Cache Miss:** - Resolve download URL using Resolver - Fetch artifact from upstream using Fetcher - - Copy the body to Storage and, for a client GET without a `Range` - header, to the client at once (`storeWhileServing`); the client's copy - is checked against the upstream's digest as it is read, and a mismatch - aborts it. A block-mode scanner, `HEAD` and range requests, and - requests joining a fetch already running wait for the stored file. - Store artifact in Storage (returns size, hash) - Create/update database records (package, version, artifact) - - Requests that waited open the stored file and stream it to their client - -Artifact handlers honor single byte ranges when the storage reader supports -efficient seeking (currently the local filesystem reader). They advertise -`Accept-Ranges: bytes`, return `206` or `416` as appropriate, and respect -`If-Range`. Malformed and multi-range requests fall back to the full response. -Non-seekable storage readers do not advertise range support, and direct-storage -redirects rely on the destination's capabilities. Range responses cannot verify -the full artifact digest because they read only part of the artifact. + - Open stored file + - Return reader to handler + - Handler streams file to client ``` ┌────────┐ GET /npm/lodash/-/lodash-4.17.21.tgz ┌─────────────┐ @@ -101,8 +88,6 @@ the full artifact digest because they read only part of the artifact. SQLite or PostgreSQL database for cache metadata. SQLite uses `modernc.org/sqlite` (pure Go, no CGO). PostgreSQL uses `lib/pq`. -For SQLite, `DB` holds two pools: one connection for writes and a pool of read-only (`query_only`) connections for reads. `Get`, `Select`, `Query` and `QueryRow` go to the reader; `Exec`, transactions and everything else go to the writer, so a read inside a transaction sees the transaction's writes. A write sent to the reader fails instead of running. WAL mode lets the readers run while the writer commits. PostgreSQL and in-memory SQLite use one pool for both. - The schema is compatible with [git-pkgs](https://github.com/git-pkgs) databases. The proxy adds the `artifacts` and `vulnerabilities` tables on top of the shared `packages` and `versions` tables, so both tools can point at the same database. **Tables:** @@ -161,11 +146,6 @@ artifacts ( ) -- indexes: (version_purl, filename) unique, storage_path, last_accessed_at -pending_deletes ( - path TEXT NOT NULL PRIMARY KEY, -- storage path no record points at - queued_at DATETIME NOT NULL -) - vulnerabilities ( id INTEGER PRIMARY KEY, vuln_id TEXT NOT NULL, -- e.g. CVE-2021-1234 @@ -189,26 +169,14 @@ metadata_cache ( storage_path TEXT NOT NULL, etag TEXT, content_type TEXT, - content_encoding TEXT, -- replayed on serve so signed bytes stay verbatim size INTEGER, -- BIGINT on Postgres fetched_at DATETIME, created_at DATETIME, updated_at DATETIME ) -- indexes: (ecosystem, name) unique - -metadata_dates ( - ecosystem TEXT NOT NULL, - proxy_url TEXT NOT NULL, -- each proxy URL rewrites to its own download URLs - name TEXT NOT NULL, - etag TEXT NOT NULL, -- the rewritten representation being dated - last_modified INTEGER NOT NULL, -- Unix seconds; BIGINT on Postgres - PRIMARY KEY (ecosystem, proxy_url, name) -) ``` -`metadata_dates` holds the `Last-Modified` served with rewritten metadata, such as npm packuments, Composer documents, PyPI simple pages and Helm indexes. A package's date is the second it started serving its current representation, and a new representation is only dated in a later second. Keeping it in the database means replicas sharing one serve the same date, and never give one representation's date to another, so `If-Modified-Since` cannot return 304 for a body the client does not have. - On PostgreSQL, `INTEGER PRIMARY KEY` becomes `SERIAL`, `DATETIME` becomes `TIMESTAMP`, `INTEGER DEFAULT 0` booleans become `BOOLEAN DEFAULT FALSE`, and size/count columns use `BIGINT`. The `MigrateSchema()` function handles backward compatibility with older git-pkgs databases by running named migrations that add missing columns and tables. See [migrations.md](migrations.md) for how to add new schema changes. @@ -240,10 +208,10 @@ type Storage interface { ``` **Filesystem implementation:** -- Stores files in nested directories: `{ecosystem}/{name}/{version}/{fetch id}/{filename}`, a new id per fetch, so fetches of one artifact never overwrite or delete each other's object (artifacts cached before this sit at `{ecosystem}/{name}/{version}/{filename}`) +- Stores files in nested directories: `{ecosystem}/{name}/{version}/{filename}` - Atomic writes using temp file + rename - Computes SHA256 hash during write -- Removes a fetch's directory once its object is deleted +- Cleans up empty parent directories on delete **Path structure:** @@ -252,18 +220,15 @@ cache/artifacts/ ├── npm/ │ ├── lodash/ │ │ └── 4.17.21/ -│ │ └── 3f9a0c1d2e4b5a67/ -│ │ └── lodash-4.17.21.tgz +│ │ └── lodash-4.17.21.tgz │ └── @babel/ │ └── core/ │ └── 7.23.0/ -│ └── 8c2e41f09a7d3b15/ -│ └── core-7.23.0.tgz +│ └── core-7.23.0.tgz └── cargo/ └── serde/ └── 1.0.193/ - └── d05b7e9c14a2f863/ - └── serde-1.0.193.crate + └── serde-1.0.193.crate ``` ### `internal/upstream` @@ -275,8 +240,6 @@ Fetches artifacts from upstream registries. - Exponential backoff retry on 429 (rate limit) and 5xx errors - Returns streaming reader (doesn't load into memory) - Configurable user-agent -- Shares an authentication-aware transport with metadata requests so URL-scoped credentials apply consistently -- Discovers and caches scoped OCI Bearer tokens from registry challenges **Resolver:** - Determines download URL for a package/version @@ -297,47 +260,13 @@ HTTP protocol handlers for each registry type. **NPMHandler:** - `handlePackageMetadata()` - Proxy + rewrite metadata - `handleDownload()` - Serve cached artifact -- Fetches, caches and serves packuments, and caches their rewrites; the rewrite itself is `internal/metadata/npm` +- Rewrites tarball URLs to point at proxy **CargoHandler:** - `handleConfig()` - Return registry config - `handleIndex()` - Proxy sparse index - `handleDownload()` - Serve cached crate -**DebianHandler:** -- Serves `/debian` and `/debian/{name}` mirrors of APT archives (`upstream.debian_repositories`) -- Routes `.deb` packages to the artifact cache, `dists/…` to the metadata cache, and proxies all other paths -- Keys the artifact cache on name, version, and filename, scoped by repository for named archives - -**SwiftHandler:** -- Proxies the Swift Package Registry v1 read endpoints -- Rewrites release URLs and caches source archives -- With `cache_metadata`, caches release metadata, release lists and manifests (with their `Link` header) and serves them when the upstream fails - -**GitHandler:** -- Serves `/git/{name}/{repo}` from `internal/gitcache`, read-only over smart HTTP -- Validates repository paths and the `repos` allowlist, and maps cache errors to statuses -- Serves the Git LFS API under `{repo}.git/info/lfs/`: forwards download batches upstream and points each object at `objects/{oid}/{size}`, which is served from the artifact cache (ecosystem `git`, name `{name}/{repo}`, version the object ID); a miss uses the download URL kept from the batch or asks the upstream batch API for a fresh one, recorded without its signed query, copies the object from another repository when its content is already stored, and checks the object against its sha256 ID; a batch the upstream can't answer is answered from the cache - -### `internal/metadata/npm` - -The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. `ScanPackument` and `RewriteStream` do the same rewrite over a reader in two passes, the first collecting the time map, dist-tags and version names and the second copying each kept version, so the handler can rewrite a packument from storage without holding it in memory. It imports no database, storage, metrics, HTTP or logging packages, which a test checks. - -### `internal/jsonscan` - -Reads and rewrites members of a JSON document in place, by byte offsets, without decoding the whole document. Shared by the npm and Composer metadata rewrites. - -### `internal/gitcache` - -Local bare clones of upstream Git repositories, outside the artifact cache and the database, at `{git.cache_dir}/{name}-{URL hash}/{repo}.git`. The hash of the upstream's URL means repointing an upstream starts a fresh cache; at startup only the configured upstreams' directories are loaded, and those of their old URLs removed. - -- Each request for refs checks the upstream's branches and tags with protocol v2 `ls-refs` and fetches only when they changed; concurrent requests share one refresh -- Clients are served by `git upload-pack --stateless-rpc` -- Spawned `git` reaches the upstream only through a loopback relay that forwards with the proxy's upstream client, so address checks, `upstream.auth` and the access log apply -- Commits a client asks for by ID that are on no cached branch or tag are fetched and kept under hidden `refs/proxy/wanted/{time}/{id}` refs until a branch or tag includes them or for 7 days; off for upstreams with credentials -- Evicts least recently used repositories after each fetch once `git.max_size` is passed, and drops a repository that can no longer be refreshed within `git.max_repo_size` so it is cloned again; never one in use. A commit fetched by ID that does not fit is refused instead -- Every fetched pack is kept as a pack (`transfer.unpackLimit=1`), so a transfer cut off at the size limit leaves nothing behind - ### `internal/server` HTTP server setup, web UI, and API handlers. @@ -348,11 +277,7 @@ HTTP server setup, web UI, and API handlers. - Web UI under `/ui`: dashboard, package browser, source browser, version comparison - Templates are embedded in the binary via `//go:embed` - Enrichment API for package metadata, vulnerability scanning, and outdated detection -- Health, stats, and Prometheus metrics endpoints. `/health` runs an active write → size-check → read → verify → delete probe against the storage backend and returns a structured JSON response (`HealthResponse`) with `"ok"` / `"error"` status per subsystem. Probe results are cached (default 30 s, configurable via `health.storage_probe_interval`) to avoid overwhelming remote backends. The response also carries a `circuit_breakers` map reporting each upstream's artifact-fetch breaker as `"open"` or `"closed"`, keyed by the host fetched from (or an opaque placeholder where the fetch URL has no host to read); the same state is published as the `proxy_circuit_breaker_state` gauge on each `/metrics` scrape. An open breaker leaves the overall status `"ok"` — it describes an upstream, not this proxy. - -### `internal/memlimit` - -Sets Go's soft memory limit to 80% of the cgroup memory limit at startup on Linux, unless `GOMEMLIMIT` is set. See "Memory limit" in the configuration docs. +- Health, stats, and Prometheus metrics endpoints. `/health` runs an active write → size-check → read → verify → delete probe against the storage backend and returns a structured JSON response (`HealthResponse`) with `"ok"` / `"error"` status per subsystem. Probe results are cached (default 30 s, configurable via `health.storage_probe_interval`) to avoid overwhelming remote backends. ### `internal/metrics` @@ -360,22 +285,18 @@ Prometheus metrics for cache performance, upstream latency, storage operations, ### Cooldown -Version age filtering for supply chain attack mitigation, provided by [github.com/git-pkgs/cooldown](https://github.com/git-pkgs/cooldown). Configurable at global, ecosystem, and per-package levels. Supported by the npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm handlers. +Version age filtering for supply chain attack mitigation, provided by [github.com/git-pkgs/cooldown](https://github.com/git-pkgs/cooldown). Configurable at global, ecosystem, and per-package levels. Supported by npm, PyPI, pub.dev, and Composer handlers. ### `internal/enrichment` -Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://github.com/git-pkgs/enrichment). Fetches license, description, homepage, repository URL, and vulnerability data from upstream registries. Powers the `/api/` endpoints and the web UI's package detail pages. +Package metadata enrichment. Fetches license, description, homepage, repository URL, and vulnerability data from upstream registries. Powers the `/api/` endpoints and the web UI's package detail pages. ### `internal/mirror` -Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and the commit or tag, with its submodules, in the git cache that `/git` serves. +Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. -### `internal/inventory` - -Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated marker on the project pages. - ### `internal/config` Configuration loading. @@ -413,8 +334,6 @@ Eviction can be implemented as: 2. When over limit, get LRU artifacts 3. Delete from storage and clear database records -An object a record stops pointing at, because a refetch replaced it or its entry was discarded, is not deleted at once: a request that read the record may still be opening it. Its path goes into `pending_deletes`, and a background loop deletes it after a grace period of at least an hour, or `direct_serve_ttl` if longer, so signed URLs to it stay valid. This runs whether or not `max_size` is set. - ## Design Decisions **Why SQLite?** @@ -432,7 +351,6 @@ An object a record stops pointing at, because a refetch replaced it or its entry - Fresh data - new versions visible immediately - Metadata is small, upstream fetch is fast - Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table -- OCI manifests and tag lists are exceptions: they are cached automatically so previously fetched images remain pullable and tag resolution works when the registry or token service is unavailable **Why stream artifacts?** - Memory efficient - don't load large files into RAM diff --git a/docs/configuration.md b/docs/configuration.md index d887509..dcdec24 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,8 +2,6 @@ The proxy can be configured via command line flags, environment variables, or a configuration file. Command line flags take precedence over environment variables, which take precedence over the configuration file. -Boolean environment variables accept `true`, `t`, `1`, `yes` or `on`, and `false`, `f`, `0`, `no` or `off`, in any case. Any other value, such as a typo, is ignored: the setting keeps its value from the configuration file or default, and the proxy logs a warning at startup. - ## Configuration File Create a YAML or JSON file and pass it with `-config`: @@ -21,29 +19,6 @@ See `config.example.yaml` in the repository root for a complete example. | `listen` | `PROXY_LISTEN` | `-listen` | `:8080` | Address to listen on | | `base_url` | `PROXY_BASE_URL` | `-base-url` | `http://localhost:8080` | Public URL package managers use to reach this proxy | | `ui_base_url` | `PROXY_UI_URL` | - | (defaults to `base_url`) | Public URL where the web UI is reached. Set separately when the UI lives behind a different hostname than package endpoints (e.g. public domain vs Docker network alias). Used for canonical/og:url tags and the install guide banner. The proxy still serves package endpoints on the same listener, so any reverse proxy fronting the UI publicly should restrict the public route to `PathPrefix(/ui)` to avoid exposing package endpoints. | -| `trust_forwarded_for` | `PROXY_TRUST_FORWARDED_FOR` | - | `false` | Attribute requests to the leftmost `X-Forwarded-For` entry instead of the TCP peer address, in the structured log, the access log and the request-source table on `/ui/analytics`. | -| `ui_request_sources` | `PROXY_UI_REQUEST_SOURCES` | - | `false` | Show the request-source table on `/ui/analytics`, which reports caller addresses, the tool each ran and how much each pulled. | -| `ui_failure_details` | `PROXY_UI_FAILURE_DETAILS` | - | `false` | List the latest failures behind each failure count on `/ui/analytics`: the package, its upstream URL or storage path, and the error. URLs are shown without credentials or query strings. | -| `http_timeout` | `PROXY_HTTP_TIMEOUT` | - | `30s` | Timeout for upstream metadata and pass-through requests. | -| `write_timeout` | `PROXY_WRITE_TIMEOUT` | - | `5m` | Response write deadline, including download and scanning time before the response starts. | -| `artifact_timeout` | `PROXY_ARTIFACT_TIMEOUT` | - | `5m` | Timeout for each upstream artifact download while serving requests; excludes subsequent scanning. Does not limit `proxy mirror`. | - -These timeouts accept Go durations such as `30s`, `5m`, or `1h`. An omitted value uses the default; `"0"` disables that timeout. Invalid or negative values are rejected at startup. Environment variables override file values. - -For long scans, increase `write_timeout` to cover the download, scan, and response transfer. `scanning.timeout` controls the scan separately; changing it alone does not extend the response deadline. `artifact_timeout` is independent of both. For example: - -```yaml -write_timeout: "18m" -artifact_timeout: "10m" -scanning: - timeout: "16m" -``` - -These are independent limits, not reserved time slices: a download approaching 10 minutes followed by a 16-minute scan would still exceed an 18-minute response deadline. Any reverse proxy in front of the server also needs sufficient timeout allowance. The upstream transport's existing response-header timeout remains unchanged. - -### Memory limit - -On Linux, when the process runs under a cgroup memory limit, such as a container's `--memory` or a Kubernetes memory limit, `proxy serve` sets Go's soft memory limit to 80% of it and log the value at startup (`set Go memory limit from cgroup`). The garbage collector then runs more often as memory use approaches the limit, rather than letting the heap grow until the container is killed. The remaining 20% is left for memory Go doesn't manage, such as `git` processes the proxy runs. Set the `GOMEMLIMIT` environment variable to choose the value yourself, or `GOMEMLIMIT=off` to disable it. ## Storage @@ -67,98 +42,7 @@ storage: |--------|-------------|------|-------------| | `storage.url` | `PROXY_STORAGE_URL` | `-storage-url` | Storage URL (file:// or s3://) | | `storage.path` | `PROXY_STORAGE_PATH` | `-storage-path` | Local path (deprecated, use url) | -| `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB"). Negative or out-of-range sizes are rejected at startup | -| `storage.cache_artifacts` | `PROXY_STORAGE_CACHE_ARTIFACTS` | - | Store fetched artifacts (default: true); `false` streams them from upstream | -| `storage.retention.default` | `PROXY_STORAGE_RETENTION_DEFAULT` | - | Evict artifacts not served for this long (e.g. "30d"); see [Retention](#retention) | -| `storage.retention.sweep_interval` | `PROXY_STORAGE_RETENTION_SWEEP_INTERVAL` | - | How often expired artifacts are looked for (default: "10m", at least "1m") | - -`storage.max_size` counts cached artifacts and cached metadata documents (`cache_metadata`). When the total goes over the limit, the least recently served entries of either kind are evicted first; an evicted metadata document is fetched from upstream again on the next request for it. Pushed artifacts are not counted or evicted. An artifact replaced by a refetch stays in storage for at least an hour, or `storage.direct_serve_ttl` if longer, so requests already reading it can finish, and storage use can exceed the limit by what was replaced in that time. - -### Cache misses - -On a cache miss the proxy sends the artifact to the client while it stores it, so the client gets its first bytes as soon as the upstream sends them, rather than after the whole artifact is stored. Clients that give up after a while without data, such as git-lfs (30 seconds) or pacman (10 seconds), no longer time out on large artifacts. The response goes out chunked, without `Content-Length`, as with `cache_artifacts: false`; once stored, the artifact is served with its size. - -The bytes are checked against the digest the upstream declares, where it declares one, as they are sent. On a mismatch the connection is aborted before the response completes, so the client never receives a tampered artifact as a good one, and nothing is cached. The upstream is read at its own speed whatever the client's: a client more than 8 MB behind continues from the stored copy once it is stored, so a slow client doesn't hold back the download, other requests waiting for it, or `artifact_timeout`. If the client goes away, the artifact is still stored. If storing it fails, for example because the disk is full, the client still gets the whole artifact; the failure is logged at error level and counted in `proxy_artifacts_served_uncached_total`, so a cache that has stopped filling doesn't go unnoticed while every request succeeds. - -Requests for the same artifact that arrive while it is being stored wait for the stored copy, as do `HEAD` requests and requests with a `Range` header, which are answered from it with its size. With a `block`-mode scanner configured for the ecosystem, a miss is stored and scanned before anything is sent, as before (see [Artifact Scanning](#artifact-scanning)). - -### Serving artifacts without storing them - -With `storage.cache_artifacts: false` the proxy streams every artifact download from upstream to the client and stores no artifacts. Metadata is still filtered and cached, so cooldown and the denylist apply as usual, and the database still holds the publish times cooldown needs. Use it when another caching layer, such as an Artifactory remote repository, sits in front of the proxy and caching artifacts twice only costs storage. - -```yaml -storage: - cache_artifacts: false -``` - -Every download is a fresh upstream fetch, and concurrent requests for the same artifact are not combined. Artifacts with a digest known up front (OCI blobs, Swift archives, Helm charts) are verified while streaming: the response is sent chunked, and on a mismatch the connection is aborted before the response completes so the client never receives a tampered artifact as a good one. The same happens when the upstream connection fails mid-download. - -`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve`, `mirror_api` or `mirror.schedules`, which all need stored artifacts, and the `mirror` command refuses to run with it. - -### Retention - -Retention evicts cached artifacts that nobody has downloaded for a while. It works alongside `storage.max_size`: retention removes what has gone unused, and the size limit still applies to whatever is left. - -The `ecosystems` and `packages` entries below only pass validation once the ecosystems they name support retention; see the table further down. - -```yaml -storage: - retention: - default: "30d" - ecosystems: - npm: "14d" - maven: "0" - packages: - "pkg:npm/lodash": "0" - sweep_interval: "10m" -``` - -Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. The blobs of an image that `mirror.schedules` keeps current also count from the last run that found them cached, so the schedule keeps them; for `max_size` eviction such a run doesn't count as use. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`. - -A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version, written in one of the forms the table below lists for the ecosystem. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems, except `generic`, which requires an explicit rule. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention supports (`covered`) and which ones `default` does not reach (`default_not_applied`), including `generic`. - -Generic retention is opt-in because release assets may disappear upstream and cannot then be downloaded again after eviction. An existing `default` never starts evicting generic downloads. Set `ecosystems.generic: "30d"` to cover all generic artifacts, or configure only selected GitHub repositories: - -```yaml -storage: - retention: - default: "30d" # Other supported ecosystems; never generic. - packages: - "pkg:generic/jqlang%2Fjq": "30d" -``` - -The package rule covers every release and asset in that repository, but each cached file ages separately. Other generic repositories remain exempt from age-based eviction. A package rule of `"0"` also exempts that repository when `ecosystems.generic` is enabled. The independent `storage.max_size` limit still applies. - -| Ecosystem key | Retention supported | Package rule form | -|---------------|---------------------|-------------------| -| `alpine` | yes | `pkg:apk/curl` or `pkg:apk/alpine/curl`; case doesn't matter. A rule covers the package in every configured APK repository. Alpine mirrors keep only the current build of each package per release, so an evicted older one can't be fetched again. | -| `cargo` | yes | `pkg:cargo/serde`, the crate name as published; case, `-` and `_` have to match, so `pkg:cargo/inflector` doesn't cover `Inflector`. | -| `composer` | yes | `pkg:composer/symfony/console`; case doesn't matter. A key without a vendor is refused. | -| `conan` | yes | `pkg:conan/zlib`, the recipe name; a rule covers every version, revision and user/channel of it. | -| `conda` | yes | `pkg:conda/conda-forge/numpy`, with the channel as the namespace; the package name's case doesn't matter. A key without a channel is refused. | -| `cran` | yes | `pkg:cran/ggplot2`, the package name as published, case included (`pkg:cran/Rcpp`). A rule covers source and binary builds alike. | -| `deb` | yes | `pkg:deb/curl`; case doesn't matter. A vendor namespace such as `pkg:deb/debian/curl` is accepted and dropped, so a rule covers the package from every configured archive. Debian and Ubuntu archives only keep the current version per suite, so an evicted older one can't be fetched again. | -| `freebsd` | yes | `pkg:bsd/freebsd/zsh`, the package name as published, case included (`pkg:bsd/freebsd/FreeBSD-acct`). A rule covers the package in every configured repository and ABI. Only packages under `Hashed/` are cached, and mirrors keep only the current build, so an evicted older one can't be fetched again. | -| `gem` | yes | `pkg:gem/rails`, the gem name as published, case included. | -| `generic` | yes, explicit rules only | `pkg:generic/owner%2Frepo` for a GitHub Release repository such as `jqlang/jq`. The ecosystem rule covers all cached version-pinned generic downloads, including GitHub Release assets and configured `generic_artifacts`. The global `default` never applies; metadata/API responses are not artifact retention candidates. | -| `golang` | yes | `pkg:golang/github.com/stretchr/testify`, the module path as in `go.mod`; case doesn't matter. A major version like `/v2` (or `.v3` on gopkg.in) is a module of its own and needs its own rule. The `!`-escaped form from proxy URLs and the module cache (`github.com/!azure/...`) is refused. | -| `helm` | yes | No package rules: charts are cached per repository and digest, not under the chart name. The `helm` rule and `default` cover charts from HTTP repositories; charts pulled over OCI fall under `oci`. | -| `hex` | yes | `pkg:hex/phoenix`, the package name as published. | -| `julia` | yes | No package rules: a julia PURL needs a `uuid` qualifier, which the cache doesn't store. The `julia` rule and `default` cover package tarballs, artifacts and registry snapshots alike. | -| `maven` | yes | `pkg:maven/com.google.guava/guava`, group and artifact as published, case included. Gradle build cache entries are not covered; they use `gradle.build_cache.max_age`. | -| `npm` | yes | `pkg:npm/lodash`, `pkg:npm/@babel/core`; the name has to match exactly, case included, so `pkg:npm/jsonstream` doesn't cover `JSONStream`. | -| `nuget` | yes | `pkg:nuget/Newtonsoft.Json`; case doesn't matter. | -| `oci` | yes | `pkg:oci/library/nginx`, the repository name of the pull. A named upstream keeps its prefix, also when containerd reaches it through `ns`: `ghcr.io/owner/app` pulled that way is `pkg:oci/upstream/ghcr/owner/app`. Homebrew bottles use Homebrew's image name, with `@` written as `/` and `+` as `x` (`openssl@3` is `pkg:oci/homebrew/core/openssl/3`). Helm charts pulled over OCI are stored like images. Only blobs are evicted; manifests stay in the metadata cache. | -| `pub` | yes | `pkg:pub/http`, the package name as published. | -| `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. | -| `rpm` | yes | `pkg:rpm/curl`, the name as packaged, case included (`pkg:rpm/NetworkManager`). A vendor namespace such as `pkg:rpm/fedora/curl` is accepted and dropped. Fedora and EPEL update repositories keep only the latest build of a package, so an evicted older one may not be fetchable again. | -| `swift` | yes | `pkg:swift/apple/swift-nio`, with the registry scope as the namespace, or the stored form `pkg:generic/swift-registry/apple.swift-nio`; case doesn't matter. A source repository key like `pkg:swift/github.com/apple/swift-nio` is looked up through `upstream.swift`'s `/identifiers` before each sweep and applies to every identity listed for it; until every such key resolves, Swift artifacts aren't swept, so a rule that keeps a package is never skipped. | - -An evicted artifact is fetched again from upstream on the next request. Packages preloaded with `proxy mirror` age from the time they were mirrored, so an unused mirrored package is evicted like any other. Running `proxy mirror` again over a package that is still cached counts as a download and restarts its clock, so packages a scheduled mirror job keeps listing never age out. A registry that removes old releases (Linux distribution archives in particular) cannot serve an evicted version again, so set those ecosystems' rules with that in mind. - -Expired artifacts are not deleted right away. Their records are cleared and the files wait in the same queue a replaced artifact goes through, for at least an hour (or `storage.direct_serve_ttl` if longer), so downloads already in progress can finish. `storage.max_size` counts the space as free from the moment the record is cleared. Once that wait is over, each minute the queue is worked through for up to 30 seconds, pausing until the next minute after a delete fails. One sweep clears at most 10,000 artifacts, so the first sweep over a large, old cache spreads its evictions, and the downloads that bring popular artifacts back, over several intervals. - -With several proxies sharing one database and `database.hit_flush_interval` set, a proxy's sweep cannot see downloads another proxy has not written yet. Keep retention durations well above the flush interval. +| `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB") | ### Amazon S3 @@ -175,8 +59,6 @@ export AWS_SECRET_ACCESS_KEY=your-secret export AWS_REGION=us-east-1 ``` -Objects up to 16 MiB are buffered in memory and sent in one request. Larger ones are uploaded in 5 MiB parts, two at a time, so a large artifact being cached holds about 30 MiB of upload buffers. Azure Blob Storage uploads use the same part size and concurrency. - ### S3-Compatible (MinIO, etc.) ```yaml @@ -201,10 +83,6 @@ database: | `database.driver` | `PROXY_DATABASE_DRIVER` | `-database-driver` | `sqlite` or `postgres` | | `database.path` | `PROXY_DATABASE_PATH` | `-database-path` | SQLite file path | -The database runs in WAL mode. Writes go through a single connection, since SQLite allows one writer at a time. Reads use a separate pool of read-only connections (one per CPU, between 2 and 8), so request lookups don't wait behind writes, stats queries or eviction scans. An in-memory database (`:memory:`) uses one connection for both. - -The SQLite database needs a local disk or a block-backed volume, even with one replica. SQLite's WAL mode doesn't work on network filesystems such as NFS ([sqlite.org/wal.html](https://www.sqlite.org/wal.html)), which shows up as stalled requests and `database disk image is malformed`. A Kubernetes `ReadWriteOnce` volume doesn't guarantee suitable storage, since its storage class can still be NFS. If only network filesystem storage is available to the proxy, use `driver: postgres`. Artifact storage can stay on NFS, on a separate volume from the SQLite database. - ### PostgreSQL ```yaml @@ -217,37 +95,6 @@ database: |--------|-------------|------|-------------| | `database.url` | `PROXY_DATABASE_URL` | `-database-url` | PostgreSQL connection URL | -### Several instances - -Several instances can serve one cache when they share a PostgreSQL database and storage. The storage can be S3 or a `file://` directory on a volume every instance mounts, such as NFS, CephFS or a Kubernetes `ReadWriteMany` volume. Each fetch writes its own file and renames it into place, and records are updated with compare-and-swap, so concurrent fetches and a crashed instance leave no partial files or records that don't match a file. SQLite only supports a single instance. - -Instances coordinate through PostgreSQL advisory locks: - -- Instances started together create the schema and run migrations one after the other. -- Eviction, reclaim, retention and the Gradle build cache sweep run on one instance at a time. An instance that finds a job running elsewhere skips that pass. -- A [scheduled mirror](#scheduled-refresh) runs on whichever instance reaches it first in each interval. - -Cache misses are fetched from upstream once across instances. The instance that fetches holds a lease on the miss in the `fetch_leases` table and renews it while it downloads. Another instance missing the same file waits until it is stored and serves the stored copy; `proxy_fetch_lease_waits_total` counts those waits. If the fetch fails, the instances waiting on it stop waiting together: each serves its stale metadata copy if it has one, and otherwise fetches itself, so a failing upstream costs every instance one attempt at the same time rather than one after another. When the fetch failed because the upstream rate limited it, the waiters without a stale copy answer 503 with the upstream's `Retry-After` instead of fetching. A lease of an instance that stopped expires after 30 seconds. Metadata is shared this way only with [`cache_metadata`](#metadata-caching) and a `metadata_ttl` above zero, since without a stored copy there is nothing to wait for. A cache hit costs no extra query; a miss costs one. - -Rate-limit holds (see `upstream_holds` under [Health Check](../README.md#health-check)) are shared through the `upstream_holds` table. When one instance gets a 429, or a 503 with `Retry-After`, it records the host and the end of the hold, and the other instances stop sending to that host too instead of each waiting for its own 429. Each instance rereads the table at most every two seconds, so another instance can still send for up to two seconds after a hold starts. A hold is also recorded with SQLite, where it then outlasts a restart. - -Each instance needs its own [`git.cache_dir`](#git-repositories). - -### Hit counts - -Every cache hit updates the artifact's hit count and last-access time, which the stats pages and LRU eviction use. Rather than writing each hit as its own transaction, the proxy counts hits in memory and writes them together every `hit_flush_interval`. SQLite allows one writer at a time, and the proxy writes through a single SQLite connection, so with a write per hit, concurrent downloads queue behind each other. - -```yaml -database: - hit_flush_interval: "1s" -``` - -| Config | Environment | Flag | Description | -|--------|-------------|------|-------------| -| `database.hit_flush_interval` | `PROXY_DATABASE_HIT_FLUSH_INTERVAL` | - | How often batched hits are written (default `1s`). `0` writes each hit as it happens. | - -Hits not yet written are lost if the process is killed; a normal shutdown writes them. - ## Logging ```yaml @@ -261,810 +108,22 @@ log: | `log.level` | `PROXY_LOG_LEVEL` | `-log-level` | `debug`, `info`, `warn`, `error` | | `log.format` | `PROXY_LOG_FORMAT` | `-log-format` | `text`, `json` | -## Access Log - -The optional access log records client requests and each HTTP exchange with an upstream registry. It is always written as JSONL, with one JSON object per line. Records for the same client request share a `request_id`. - -```yaml -access_log: - path: "/var/log/proxy/access.jsonl" -``` - -| Config | Environment | Flag | Description | -|--------|-------------|------|-------------| -| `access_log.path` | `PROXY_ACCESS_LOG_PATH` | `-access-log` | File to append JSONL records to; empty disables the log | - -Each client request record carries `remote_addr` (the TCP peer, host and port), `remote_ip` (the address the request is attributed to), `user_agent`, `client` (the tool name derived from the User-Agent), `ecosystem`, and `bytes` (the response body size written to the client). The structured log carries the same `client`, `remote` and `remote_ip` fields. - -`remote_ip` follows the top-level `trust_forwarded_for` setting. Enable that only when the proxy sits behind a load balancer or ingress that sets the header: any client can send `X-Forwarded-For`, so behind such a hop it is the only way to see the real caller, but in front of one it lets a caller choose what address it is logged as. It can also choose a fresh one per request, which fills the bounded source table on `/ui/analytics` and evicts the genuine callers from it. `remote_addr` is unaffected and always records the TCP peer. - -The parent directory must exist and be writable when the proxy starts. A newly created log file is readable and writable only by the proxy process owner. - -A request that receives a rate limit response from an upstream can produce records like these: - -```json -{"time":"2026-08-16T12:00:00Z","event":"upstream","request_id":"host/example-000001","method":"GET","url":"https://registry.example/packages/example","status_code":429,"duration_ms":42} -{"time":"2026-08-16T12:00:00Z","event":"request","request_id":"host/example-000001","method":"GET","path":"/npm/example","status_code":502,"duration_ms":43,"remote_addr":"192.0.2.10:41234"} -``` - -Upstream retries and OCI authentication calls are separate `upstream` records, so the log preserves every status returned over the wire. Network failures have an `error` field and no `status_code`. URL credentials, query strings, and fragments are omitted from both upstream URLs and client paths. - ## Upstream Registries -Each upstream used by a built-in package route can be set in YAML or JSON under `upstream`, or with its matching environment variable. Existing installations keep the same public upstreams by default. Trailing slashes are ignored. - -| Config | Environment | Default | -|--------|-------------|---------| -| `upstream.allow_private_hosts` | `PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS` | `[]` | -| `upstream.allow_loopback` | `PROXY_UPSTREAM_ALLOW_LOOPBACK` | `false` | -| `upstream.npm` | `PROXY_UPSTREAM_NPM` | `https://registry.npmjs.org` | -| `upstream.npm_full_metadata` | `PROXY_UPSTREAM_NPM_FULL_METADATA` | `false` | -| `upstream.npm_change_feed` | `PROXY_UPSTREAM_NPM_CHANGE_FEED` | `false` (see [change feeds](#change-feeds)) | -| `upstream.cargo` | `PROXY_UPSTREAM_CARGO` | `https://index.crates.io` | -| `upstream.cargo_download` | `PROXY_UPSTREAM_CARGO_DOWNLOAD` | `https://static.crates.io/crates` | -| `upstream.gem` | `PROXY_UPSTREAM_GEM` | `https://rubygems.org` | -| `upstream.go` | `PROXY_UPSTREAM_GO` | `https://proxy.golang.org` | -| `upstream.go_sumdbs_offline` | `PROXY_UPSTREAM_GO_SUMDBS_OFFLINE` | `false` | -| `upstream.go_vulndb` | `PROXY_UPSTREAM_GO_VULNDB` | `https://vuln.go.dev` | -| `upstream.hex` | `PROXY_UPSTREAM_HEX` | `https://repo.hex.pm` | -| `upstream.hex_api` | `PROXY_UPSTREAM_HEX_API` | `https://hex.pm` | -| `upstream.pub` | `PROXY_UPSTREAM_PUB` | `https://pub.dev` | -| `upstream.pypi` | `PROXY_UPSTREAM_PYPI` | `https://pypi.org` | -| `upstream.pypi_download` | `PROXY_UPSTREAM_PYPI_DOWNLOAD` | `https://files.pythonhosted.org` | -| `upstream.maven` | `PROXY_UPSTREAM_MAVEN` | `https://repo1.maven.org/maven2` | -| `upstream.gradle_plugin_portal` | `PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL` | `https://plugins.gradle.org/m2` | -| `upstream.nuget` | `PROXY_UPSTREAM_NUGET` | `https://api.nuget.org` | -| `upstream.nuget_search` | `PROXY_UPSTREAM_NUGET_SEARCH` | `https://azuresearch-usnc.nuget.org` | -| `upstream.composer` | `PROXY_UPSTREAM_COMPOSER` | `https://packagist.org` | -| `upstream.composer_repository` | `PROXY_UPSTREAM_COMPOSER_REPOSITORY` | `https://repo.packagist.org` | -| `upstream.composer_change_feed` | `PROXY_UPSTREAM_COMPOSER_CHANGE_FEED` | `false` (see [change feeds](#change-feeds)) | -| `upstream.conan` | `PROXY_UPSTREAM_CONAN` | `https://center.conan.io` | -| `upstream.chef` | `PROXY_UPSTREAM_CHEF` | `https://supermarket.chef.io` | -| `upstream.conda` | `PROXY_UPSTREAM_CONDA` | `https://conda.anaconda.org` | -| `upstream.cran` | `PROXY_UPSTREAM_CRAN` | `https://cloud.r-project.org` | -| `upstream.julia` | `PROXY_UPSTREAM_JULIA` | `https://pkg.julialang.org` | -| `upstream.swift` | `PROXY_UPSTREAM_SWIFT` | `https://tuist.dev/api/registry/swift` | -| `upstream.oci_default` | `PROXY_UPSTREAM_OCI_DEFAULT` | `https://registry-1.docker.io` | -| `upstream.debian` | `PROXY_UPSTREAM_DEBIAN` | `http://deb.debian.org/debian` | -| `upstream.rpm` | `PROXY_UPSTREAM_RPM` | `https://dl.fedoraproject.org/pub/fedora/linux` | -| `upstream.homebrew_api` | `PROXY_UPSTREAM_HOMEBREW_API` | `https://formulae.brew.sh/api` | -| `upstream.homebrew_artifact` | `PROXY_UPSTREAM_HOMEBREW_ARTIFACT` | `https://ghcr.io` | -| `upstream.apk_snapshots` | `PROXY_UPSTREAM_APK_SNAPSHOTS` | disabled; see [Alpine APK snapshots](#alpine-apk-snapshots) | - -Private, ULA, CGNAT, and loopback addresses are rejected by default. Add each private upstream hostname or IP address to `upstream.allow_private_hosts`. The matching environment variable accepts a comma-separated list. Loopback upstreams also require `upstream.allow_loopback: true`. That setting permits upstream requests and redirects to reach any loopback address. - -Upstream requests honour `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY`. When a request goes through a forward proxy, these rules apply twice: to the forward proxy's own address, which therefore has to be listed in `upstream.allow_private_hosts` if it is private, and to the request's destination, which the proxy resolves before handing the request over. Link-local addresses such as `169.254.169.254` are always refused. A destination the proxy cannot resolve itself is passed to the forward proxy, since some networks only resolve public names there; the forward proxy's own policy decides what such a name may reach. +Override default upstream registry URLs: ```yaml upstream: - allow_private_hosts: - - "upstream-proxy.internal" - pypi: "http://upstream-proxy.internal/pypi" - pypi_download: "http://upstream-proxy.internal/pypi" -``` - -For protocols that use separate metadata and download services, configure both values. They may point to the same endpoint when chaining proxies: - -```yaml -upstream: - pypi: "https://upstream-proxy.example.com/pypi" - pypi_download: "https://upstream-proxy.example.com/pypi" - nuget: "https://upstream-proxy.example.com/nuget" - nuget_search: "https://upstream-proxy.example.com/nuget" - composer: "https://upstream-proxy.example.com/composer" - composer_repository: "https://upstream-proxy.example.com/composer" -``` - -The proxy reads `packages.json` from `upstream.composer_repository` and keeps it in memory for the metadata TTL, or an hour when the TTL is 0, whether or not `cache_metadata` is on. A failed fetch is not retried within that time: the last good copy is used, or the entries are left out. It passes on its `security-advisories` and `filter` entries (Packagist's malware list) with URLs pointing at the proxy. An upstream that offers neither is still usable; `composer audit` then reports nothing from it. A filter `api-url` is not passed on; Composer reads filter entries from package metadata instead. - -`upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint. - -`upstream.chef` is the Supermarket served at `/chef/`, for Berkshelf (`source "https://proxy.example.com/chef"`) and Policyfiles (`default_source :supermarket, "https://proxy.example.com/chef"`). The upstream has to serve the Supermarket API: `/universe`, `/api/v1/cookbooks/{name}`, its versions and their downloads. A private Supermarket that needs credentials takes them from `upstream.auth`, scoped to its URL. A download redirect to another host, such as the storage bucket behind Supermarket, does not receive them: - -```yaml -upstream: - chef: "https://supermarket.example.com" - auth: - "https://supermarket.example.com": - type: bearer - token: "${SUPERMARKET_TOKEN}" -``` - -### Private package routes - -Clients that use the proxy as their only registry can still install private packages. A package route sends every package whose name matches a pattern to a private registry, and never to the default upstream: - -```yaml -upstream: - composer_routes: - "example/*": "https://packages.example.com/composer" - npm_routes: - "@example/*": "https://packages.example.com/npm" - allow_private_hosts: - - "packages.example.com" - auth: - "https://packages.example.com": - type: bearer - token: "${PRIVATE_REGISTRY_TOKEN}" -``` - -- Patterns use Go's [`path.Match`](https://pkg.go.dev/path#Match) syntax and are matched case-insensitively. `*` does not cross `/`, so `example/*` matches `example/library` but not `example-fork/library`. The longest matching pattern wins. -- A Composer route URL is the repository base that serves `/p2/{vendor}/{package}.json`. A route also covers the package's dev versions in `/p2/{vendor}/{package}~dev.json`; patterns are matched against the package name without `~dev`. -- An npm route URL is the registry base that serves `/{package}`. Tarball URLs in its packuments must be on the same host and below the same path. -- A routed package is not looked up anywhere else. If the private registry does not have it, the client gets a 404. If the registry fails or rejects the credentials, the client gets a 502. A package with the same name on the public registry is never served in its place (dependency confusion). -- Metadata and artifacts of routed packages are cached under keys that include the route's URL. Entries cached from the public registry before a route was added are never served for a routed package. -- With cooldown enabled, the publish time of a routed npm or Composer version always comes from the route's metadata. Times recorded for the public package of the same name, or for a route's previous URL, are not used, and routed downloads do not record times of their own. -- Composer metadata of routed packages carries an empty `notification-url`, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. -- Use `upstream.auth` for the registry's credentials. The proxy has no authentication of its own, so every client that can reach it can download routed packages. -- Search (`/composer/search.json`) and the package list (`/composer/packages/list.json`) still come from the default upstream only. -- Composer security advisories and filter lists come from the default upstream. Routed package names, and those in `hosted.composer_vendors`, are removed from advisories requests before they are sent, so they only get advisories that their route's own metadata carries. -- Attestations of a routed npm version (`/npm/-/npm/v1/attestations/...`) come from the route, like its tarballs. -- npm audit, signing-key, search and ping requests still go to `upstream.npm`. An audit request lists every package in the dependency tree, routed ones included. -- Routes can only be set in the configuration file, not with environment variables. - -Helm HTTP repositories and additional OCI registries are configured as named maps: - -```yaml -upstream: - # Named HTTP Helm chart repositories, served at /helm/{name}/. - helm: - bitnami: "https://charts.bitnami.com/bitnami" - - # Named OCI registries. Select one with the repository prefix - # upstream/{name}/, e.g. oci://proxy.example.com/upstream/ghcr/owner/chart. - oci: - ghcr: "https://ghcr.io" -``` - -Helm HTTP repositories are read-only. The proxy fetches and rewrites each -repository's `index.yaml` so chart archives are downloaded through the proxy. -Chart archives are retained only when their SHA-256 digest matches the digest -listed in the index. Relative and absolute chart URLs are both supported. - -HTTP indexes can also contain `oci://` chart references. References matching the -host and port of a configured `upstream.oci` registry or `upstream.oci_default` -are rewritten through the OCI proxy, with named registries preferred. Unmatched -references are left unchanged, so Helm contacts those registries directly. -Automatic rewriting requires both the OCI upstream URL and `base_url` to be -root URLs without a path prefix. Repository paths, tags, and digests are preserved. - -Generic HTTP upstreams proxy plain downloads from fixed base URLs: - -```yaml -upstream: - # Named HTTP upstreams, served at /generic/{name}/. The rest of the - # request path and the query string are appended to the upstream URL. - generic: - github: "https://github.com" - github-api: "https://api.github.com" - # Paths each upstream may serve. Required for an upstream that - # upstream.auth has credentials for. - generic_paths: - github-api: - - "repos/*/*/releases" - - "repos/*/*/releases/**" - - "repos/*/*/tags" - auth: - # Optional: raise the GitHub API rate limit. Scoped to this host only, - # so the token is never sent to the object store GitHub redirects to. - "https://api.github.com": - type: bearer - token: "${GITHUB_TOKEN}" -``` - -Only configured upstreams are reachable, so this is not an open HTTP proxy. -`generic_paths` limits an upstream to the paths after `/generic/{name}/` that -its patterns match; any other path answers 404. The patterns use the syntax of -`upstream.git` `repos`: `path.Match`, where `*` does not cross a `/`, and a -pattern ending in `/**` also matches every path nested below its prefix. A -path containing `%`, `?`, `#` or a backslash is refused for an upstream with -`generic_paths`. An upstream without `generic_paths` serves every path. - -When `upstream.auth` has credentials for a generic upstream, `generic_paths` -is required and the proxy refuses to start without it: everyone who can reach -the proxy can send requests with those credentials, so list only the paths -they need. For GitHub, use a fine-grained personal access token limited to -public repositories with read-only access; it raises the rate limit like any -other token but cannot read anything private, even through an allowed path. -Paths shaped like `{owner}/{repo}/releases/download/{tag}/{asset}` are -version-pinned GitHub release assets: they are stored in the artifact cache -and served from it without revalidation, including while the upstream is -down. - -Upstreams pointed at these hosts get version-pinned paths built in, whatever -the upstream is named: - -| Upstream URL | Cached as artifacts | Package name | -|---|---|---| -| `https://archive.apache.org` | `dist/{project dirs}/{version}/[binaries/]{file}`, where `{version}` starts with a digit | the project dirs, e.g. `maven/maven-3`, `maven/mvnd` | -| `https://nodejs.org` | `dist/v{version}/{file}` | the upstream name | -| `https://dl.google.com/go` | `go{version}.{os}-{arch}.tar.gz`, `.zip` and their `.sha256` | the upstream name | - -Other upstreams can declare their own version-pinned paths in -`generic_artifacts`. These are tried before any built-in ones, so a configured -pattern that overlaps a built-in decides the package name. Each pattern is a regular expression that must match the -whole path after `/generic/{name}/`, with `version` and `file` named groups -(single path segments) and an optional `name` group for the package name, -which otherwise is the upstream name. Matching downloads are cached like -release assets and listed under the `generic` ecosystem: - -```yaml -upstream: - generic: - eclipse: "https://download.eclipse.org" - generic_artifacts: - eclipse: - - 'jdtls/milestones/(?P[^/]+)/(?P[^/]+)' -``` - -Only list paths whose content never changes for a given version: a match is -never revalidated. - -Every other path is served through the metadata cache (`cache_metadata` -must be enabled for offline fallback): fresh within `metadata_ttl`, then -revalidated with the upstream's `ETag`/`Last-Modified`, and served stale with -a `Warning: 110` header when the upstream fails, refuses or rate-limits the -request. Metadata responses are buffered up to `metadata_max_size`, so keep -large mutable downloads (`releases/latest/download/...`) off this route. - -This is the cache behind [mise](https://mise.jdx.dev)'s aqua backend; see the -mise section in the README for the client-side `url_replacements`. - -`upstream.oci_default` sets the registry used by unprefixed `/v2` requests, -while `upstream.oci` selects named registries through the `upstream/{name}/` -repository prefix. For example, `oci://proxy.example.com/upstream/ghcr/owner/chart` -uses the `ghcr` registry with `owner/chart` as its repository. - -The OCI handler only serves pulls (`GET` and `HEAD`). Pushes, such as a -BuildKit `--cache-to type=registry` export, need a separate registry. - -containerd mirror requests carry the original registry host in an `ns` query -parameter (see the containerd section in the README). The proxy only looks the -host up and never connects to it: `docker.io`, `index.docker.io`, -`registry-1.docker.io` and the host of `upstream.oci_default` select the default -registry, and the host of each `upstream.oci` URL selects that named registry. -Hosts are compared case-insensitively. The scheme's default port (443 for -`https`, 80 for `http`) may be spelled out or left out in the image reference; -any other port must match exactly, so `https://registry.example:80` and -`https://registry.example` are two different registries. An -unknown host returns `404 NAME_UNKNOWN`, so containerd falls back to its next -host. Registry URLs with a path (for example an Artifactory repository path) -are not reachable through their own host in `ns`, only through -`upstream/{name}/` or through `upstream.oci_mirrors`. When two -entries share a host, the proxy logs a warning at startup; the default registry -wins, otherwise the alphabetically first name. Pulls through `ns`, -`upstream/{name}/` and unprefixed requests share the same cache entries. -Requests that combine the `upstream/{name}/` prefix with `ns`, as per-registry -containerd mirrors with `override_path = true` send them, are routed by the -prefix. They are accepted only when `ns` is the upstream's own host, a host -listed for it in `upstream.oci_mirrors`, or Docker Hub (`docker.io` and its -aliases). Docker Hub repository names have two path components and can never -start with `upstream/`, so a Docker Hub mirror behind any prefix stays -reachable. Any other host gets `404 NAME_UNKNOWN`: a containerd `_default` -mirror sends the same request for an image such as -`unconfigured.example/upstream/ghcr/owner/app`, and that pull must not be -answered from the `ghcr` upstream. - -When a named upstream mirrors another registry, for example an Artifactory -remote of `ghcr.io`, say so in `upstream.oci_mirrors`: - -```yaml -upstream: - oci: - ghcr: "https://artifactory.example.com/artifactory/api/docker/ghcr-remote" - oci_mirrors: - ghcr: ["ghcr.io"] -``` - -Each entry lists bare registry hosts as containerd sends them, optionally with -a port. A host can be listed for one upstream only. Those hosts select the -upstream for unprefixed `ns` requests and are accepted as `ns` on its -`upstream/{name}/` prefix. A host that is also the host of a configured -registry URL stays with that registry for unprefixed requests; the proxy logs a -warning at startup. - -A host that is accepted on a prefix route gives up image paths of the form -`/upstream/{name}/...` under a `_default` mirror: with `ghcr: ["ghcr.io"]`, -a pull of `ghcr.io/upstream/ghcr/owner/app` is answered with `owner/app` from -the `ghcr` upstream, because containerd sends the same request for it as a -per-registry mirror does. The same applies to the upstream's own host. -When the proxy uses plain HTTP (for example `localhost:8080`), pass -`--plain-http` to Helm OCI commands. - -```yaml -upstream: - - # Named Alpine APK repositories, served at /apk/{name}/. - apk: - alpine: "https://dl-cdn.alpinelinux.org/alpine" -``` - -Alpine APK repositories are read-only. Requests to `/apk/{name}/…` mirror the -upstream layout, e.g. `/apk/alpine/v3.22/main/x86_64/APKINDEX.tar.gz`. Indexes -(v2 `APKINDEX.tar.gz`, v3 `Packages.adb`) and detached signatures are cached -with the metadata TTL and served byte-for-byte unchanged so apk signature -verification keeps working; `.apk` packages use the shared artifact cache. -When `upstream.apk` is empty, a single repository named `alpine` pointing at -the official mirror is available; configuring any entry replaces that default. -Earlier index versions can be kept and served by date; see -[Alpine APK snapshots](#alpine-apk-snapshots). - -```yaml -upstream: - - # Named Arch Linux pacman mirrors, served at /arch/{name}/. - arch: - archlinux: "https://geo.mirror.pkgbuild.com" -``` - -Arch Linux repositories are read-only. Requests to `/arch/{name}/…` are passed -to the mirror unchanged, so both the Arch Linux layout -(`/arch/archlinux/core/os/x86_64/core.db`) and the Arch Linux ARM layout -(`/arch/archlinuxarm/aarch64/core/core.db`) work. Databases (`{repo}.db`, -`{repo}.files` and their `.tar.*` forms) are cached with the metadata TTL and -served byte-for-byte unchanged; a database's `.sig` is refreshed together with -the database. Packages and package signatures use the shared artifact cache. -When `upstream.arch` is empty, a single repository named `archlinux` pointing at -`https://geo.mirror.pkgbuild.com` is available; configuring any entry replaces -that default. Arch Linux ARM has no default: pick a regional HTTPS mirror, not -the `mirror.archlinuxarm.org` redirector. - -```yaml -upstream: - - # Named FreeBSD pkg mirrors, served at /freebsd/{name}/. - freebsd: - freebsd: "https://pkg.freebsd.org" -``` - -FreeBSD pkg repositories are read-only. Requests to `/freebsd/{name}/…` are -passed to the mirror unchanged, e.g. -`/freebsd/freebsd/FreeBSD:14:amd64/quarterly/packagesite.pkg`, so ports and -pkgbase repositories and repositories built with poudriere work. Catalogues -(`meta`, `meta.conf`, `packagesite` and `data` as `.pkg`, `.txz` or `.tzst`) -are cached with the metadata TTL and served byte-for-byte unchanged, which -keeps their embedded signatures valid. Packages under `Hashed/` use the shared -artifact cache. Other package files (`.pkg`, `.txz`, `.tzst`), unhashed ones -and the bootstrap package under `Latest/`, can be rebuilt under the same name, -so they are passed through uncached after the [denylist](#version-denylist) -check on their `{name}-{version}`. Only cached artifacts are scanned, so while -a scanner covers the `freebsd` ecosystem these files are refused with -`403 Forbidden`: serve repositories built with `pkg repo --hash` (as the -official mirror is) and bootstrap pkg without the proxy. Everything else, such -as signatures and keys, is passed through uncached. When `upstream.freebsd` is empty, a single -repository named `freebsd` pointing at `https://pkg.freebsd.org` is available; -configuring any entry replaces that default. - -### Alpine APK snapshots - -```yaml -upstream: - # Keep replaced APK indexes for 90 days. - apk_snapshots: "90d" -``` - -Alpine mirrors only keep the current build of each package, so after an -upgrade `apk add curl=8.5.0-r0` fails because the index no longer lists that -build. With `upstream.apk_snapshots` set, the proxy keeps every version of an -index (and of a detached signature) it stores, and serves them under a dated -repository name: - -``` -http://localhost:8080/apk/alpine@2026-10-01/v3.22/main -``` - -`/apk/{name}@{YYYY-MM-DD}/…` returns the index that was current at the end of -that day (UTC). The kept files are the signed upstream bytes, so apk verifies -them as usual. A day before the first stored version of an index, or a day -that has not started yet, is `404`. Today's snapshot can still change until the -day ends. Packages under a snapshot path are the same as under the plain path: -they come from the artifact cache, or from upstream if not cached; a build the -upstream has dropped and the proxy never cached is `404`. A configured -repository whose name contains `@` takes precedence over a snapshot path. - -Snapshots are recorded when an index is requested through the plain path, so -they only exist for repositories clients use. A version counts from the first -time the proxy served it, which can be up to `metadata_ttl` after upstream -published it. A version is deleted once it was replaced more than the -configured duration ago; the newest version of each index is always kept. -Durations take a `d` suffix; empty or `"0"` disables snapshots. Snapshots need -`cache_metadata`. Their files live under `_snapshots/` in storage and do not -count towards `storage.max_size`. - -Turning snapshots off, or `cache_metadata` off, leaves the kept versions in -place. To remove them, first set a short duration such as `"1h"` and let the -hourly prune delete everything but the newest version of each index. - -Snapshots keep indexes, not packages. A package listed by an old snapshot can -still be evicted by `storage.max_size` or [retention](#retention), and is then -only available while upstream still has it. Retention counts from a package's -last download, so a pinned build nobody installed for longer than the `alpine` -retention is evicted even while a snapshot lists it. Use package rules with -`"0"` for builds that must stay installable. - -### Go checksum database - -```yaml -upstream: - - # Named Go checksum databases, served at /go/sumdb/{name}/. - go_sumdbs: - sum.golang.org: "https://sum.golang.org" - - # Keep answering for a database from its cached tree, however old, when - # the database cannot be reached. For networks where clients have no - # route to the database of their own. Requires cache_metadata. - go_sumdbs_offline: false -``` - -The `go` command asks `GOPROXY` for the checksum database before contacting it -directly. The proxy answers for every name listed here and fetches lookups and -tiles from the database's URL, not from `upstream.go`. Unknown names get 404, -so the `go` command falls back to contacting that database itself. When -`upstream.go_sumdbs` is empty, `sum.golang.org` is served; configuring any -entry replaces that default. - -Once the proxy says it serves a database, the `go` command no longer contacts -that database directly. The proxy therefore only says so when it can fetch the -database's latest signed tree, or has a fresh one cached (fetched within -`metadata_ttl`). If it cannot, it answers 404 as for an unknown name, and -clients go to the database themselves, as they would without the proxy. The -check waits five seconds at most, and after a failed check the proxy waits a -minute before it tries the database again. - -Set `upstream.go_sumdbs_offline: true` when clients have no route to the -database of their own. The proxy then keeps answering for it from the cached -tree for as long as the outage lasts and serves cached lookups and tiles. -That does not mean every cached module still verifies: the `go` command -checks each lookup against tiles of the signed tree that came with it, the -database attaches a different tree to almost every lookup, and a tile no -client fetched through the proxy before cannot be served during the outage. -Lookups for modules that were never cached fail as well. Builds whose -`go.sum` already lists every module do not ask the database at all and are -not affected. This requires `cache_metadata`. - -With `cache_metadata` enabled, lookups, partial hash tiles and the latest -signed tree follow `metadata_ttl` and are served from cache when the database -is unreachable. Complete hash tiles never change and are not revalidated once -cached; a tile of the wrong size is refused and not stored. Data tiles are -always forwarded and never stored: the `go` command does not fetch them, only -mirrors and auditors do. -Without `cache_metadata` the requests are forwarded and nothing is stored. - -A 404 from the database keeps its text, which the `go` command prints to -explain why a version is missing. With `cache_metadata` enabled the text is -cut at 4 KiB; without it the response is passed on unchanged. - -If `upstream.go` points at a module proxy that serves the checksum database -itself, such as Athens, point the entry there instead, for example -`sum.golang.org: "https://athens.example/sumdb/sum.golang.org"`. - -### Go vulnerability database - -```yaml -upstream: - - # The Go vulnerability database, served at /go/vulndb. - go_vulndb: "https://vuln.go.dev" -``` - -Point govulncheck at the proxy with its `-db` flag: - -```bash -govulncheck -db https://proxy.example.com/go/vulndb ./... -``` - -The proxy passes on the files the -[database API](https://go.dev/security/vuln/database#api) defines: -`index/db.json`, `index/modules.json`, `index/vulns.json` and `ID/{id}.json`, -each also as `.json.gz`, which is what govulncheck fetches. Other paths get -404 without reaching the database. The files are stored and served -byte-for-byte: a `.json.gz` file is gzip data, not a response with -`Content-Encoding: gzip`. - -With `cache_metadata` enabled, every file follows `metadata_ttl`, entries -included, since the database updates an entry when its `modified` date -changes. A response that is not JSON (after decompressing a `.json.gz` file) -is refused and not stored. When the database cannot be reached, cached files -are served however old, with a `Warning` header. Without `cache_metadata` -the requests are forwarded and nothing is stored. - -On a network where clients cannot reach `vuln.go.dev`, only files someone -fetched through the proxy before an outage are available during it. To have -every entry cached, fetch the indexes and all entries through the proxy while -it is online, for example: - -```bash -base=https://proxy.example.com/go/vulndb -for f in db modules vulns; do curl -sfo /dev/null "$base/index/$f.json.gz"; done -curl -sf "$base/index/vulns.json" | jq -r '.[].id' | - xargs -I{} curl -sfo /dev/null "$base/ID/{}.json.gz" -``` - -### Debian archives - -```yaml -upstream: - - # The main APT archive, served at /debian/. - debian: "http://deb.debian.org/debian" - - # Additional APT archives, served at /debian/{name}/. - debian_repositories: - security: "https://security.debian.org/debian-security" -``` - -A Debian release is served by more than one archive: security updates live on -a separate host from the main archive, so `upstream.debian` alone cannot serve -a complete suite set. Requests to `/debian/{name}/…` mirror the upstream -layout, e.g. `/debian/security/dists/trixie-security/InRelease`. - -`upstream.debian_repositories` is additive. `/debian/pool/…` and -`/debian/dists/…` continue to address `upstream.debian` with unchanged cache -identities, so existing deployments and their warm caches are unaffected. A -repository name shadows the main archive's root path of the same name; `pool` -and `dists` are refused at config load for that reason, and other root paths -an archive may serve (`indices`, `project`, `doc`, `tools`) are not, so avoid -those names unless the shadowing is intended. - -This field has no environment variable, as with the other named upstream maps. -`PROXY_UPSTREAM_DEBIAN` still sets `upstream.debian`. - -### Git repositories - -Git servers configured under `upstream.git` are served read-only at -`/git/{name}/{repo}` over Git's smart HTTP protocol, from bare repositories -cached on local disk: - -```yaml -upstream: - git: - github: "https://github.com" - wikimedia: - url: "https://gerrit.wikimedia.org/r" - repos: ["mediawiki/core", "mediawiki/extensions/**"] # optional allowlist - -git: - cache_dir: /data/git # local disk, used by this process only - max_size: 50GB # evict least recently used repositories past this - max_repo_size: 5GB # refuse a repository that grows past this - refresh_wait: 5s # then serve the cached copy if the upstream check is slow - stale_max_age: 24h # never serve a copy not checked for this long -``` - -Clients reach the proxy through Git's `insteadOf` setting, which rewrites the -start of a URL before Git connects. After - -```bash -git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/ -``` - -`git clone https://github.com/git-pkgs/purl` fetches from -`http://localhost:8080/git/github/git-pkgs/purl`, while the clone's remote -stays `https://github.com/git-pkgs/purl`. - -In CI, the same setting can come from environment variables instead of a -config file (Git 2.31 or newer), next to the variables that point package -managers at the proxy. Each `GIT_CONFIG_KEY_n` and `GIT_CONFIG_VALUE_n` pair is -one `git config` setting, and `GIT_CONFIG_COUNT` says how many pairs there are. -This sends both upstreams from the example above through the proxy: - -```bash -export GIT_CONFIG_COUNT=2 -export GIT_CONFIG_KEY_0=url.http://localhost:8080/git/github/.insteadOf -export GIT_CONFIG_VALUE_0=https://github.com/ -export GIT_CONFIG_KEY_1=url.http://localhost:8080/git/wikimedia/.insteadOf -export GIT_CONFIG_VALUE_1=https://gerrit.wikimedia.org/r/ -``` - -Every process the job starts inherits these variables, so the rewrite also -applies when a package manager runs `git` for a Git dependency, as pip, -Bundler, Composer, SwiftPM, and Go for modules it fetches directly do. Those -fetches are cached too. npm downloads GitHub dependencies as tarballs over -HTTPS where it can, without `git`, so the rewrite does not reach them. - -`repos` patterns use `path.Match` syntax, where `*` does not cross a `/`; a -pattern ending in `/**` also matches every repository nested below its -prefix, so `mediawiki/**` matches `mediawiki/skins/Vector`. An upstream -without `repos` serves every repository on it, and a repository outside -`repos` answers 404. When `upstream.auth` has credentials for an upstream, -`repos` is required: everyone who can reach the proxy can read every -repository it serves, so list only those they may read. Per-client -permissions are not supported. `upstream.auth` matches credentials by URL, so -they are sent by every route to that host, not only `/git`. A `generic` -upstream at the same URL, such as the one the mise setup in the README uses, -checks `generic_paths` instead of `repos`, and needs them too; keep Git paths -like `{owner}/{repo}.git/info/refs` out of its patterns. - -Every request for a repository's refs checks the upstream's branches and tags, -and fetches only when they changed, so a clone right after a push sees the new -commit. Requests that arrive while a check or fetch is running share it. Only -branches and tags are cached and served; other refs, such as Gerrit's -`refs/changes/*`, are left out, and protocol v2 lets the server filter them -before sending. - -A commit a client asks for by object ID that is on no cached branch or tag, -such as a pull request head fetched by `actions/checkout`, is fetched from the -upstream when the client asks for it, and kept until a cached branch or tag -includes it or for 7 days. A commit the upstream could not provide, or could -not be reached for, is not asked for again for a minute, or for an hour when -it is larger than what `max_repo_size` leaves. This is off for upstreams that `upstream.auth` has -credentials for: GitHub serves a commit from any repository in a fork network -through any other, so a fetch by ID could reach repositories outside `repos`. -Any commit the cache holds can be fetched by ID, including one a force-push -left behind, as from the upstream. - -If the upstream fails, or its check of branches and tags takes longer than -`refresh_wait`, a repository checked successfully within `stale_max_age` is -served from the cache while the refresh continues. Once the check finds new -commits, the request waits for their fetch, so a clone right after a push gets -them; if the fetch fails or stalls, the cached copy is served. A -`stale_max_age` of `0` never serves a copy the upstream has not just -confirmed. A repository the upstream reports missing (404), refuses (401, 403) -or redirects is not served from the cache, also to requests that skip the ref -listing and go straight to `git-upload-pack`, until a later check succeeds. -Redirects are never followed, so point the upstream at a repository's current -location. A `git-upload-pack` request does not check the upstream itself -unless the upstream refused the repository at the last check or the copy was -last checked more than a minute past `stale_max_age` ago, so the requests of a -clone that follow its ref listing check nothing again; otherwise the -repository is checked first, as for a ref listing. - -The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it). -The `git` processes never reach the network themselves: their fetches go -through a loopback endpoint in the proxy that forwards them with the same -upstream client as every other request, so the address checks -(`allow_loopback`, `allow_private_hosts`), `upstream.auth` and the access log -apply, and credentials are never passed to `git`. - -`max_repo_size` defaults to 5GB, and `max_size` to no limit. `max_size` also -caps `max_repo_size`, since a repository larger than the whole cache would be -evicted as soon as no request was reading it. A clone or fetch that would -pass `max_repo_size` is stopped, and nothing of it is kept. A new repository -is then refused for an hour. When a refresh of one already cached does not -fit, it is served as it is and dropped once no request is reading it, so the -next request clones it again with only its branches and tags; if that is -still too large, it is refused for an hour. A commit fetched by ID that does -not fit is refused, and the repository kept. - -`max_size` counts fetches in progress as well as cached repositories, so the -cache stays within it however many fetches run at once. As a fetch receives -data, the least recently used repositories are evicted to make room, never -one a request is reading; those are evicted once released if the cache is -still over. When evicting every idle repository would not make room, none is -evicted for the fetch, and it is stopped and nothing of it is kept, but the -repository is not refused: a clone answers 507, and a refresh serves the -cached copy, as when the upstream fails. The count is of bytes received, so -index files and garbage collection can take the cache a little past -`max_size` until the fetch ends and the repository is measured. -A transfer that receives nothing for 10 minutes is abandoned. Git's automatic -garbage collection runs as part of fetches, and files over 8MB are kept -without delta compression, so one large file does not take a fetch's memory. - -Packs for clients are built by at most two `upload-pack` processes per CPU -the proxy may use, each with one delta-search thread and 64MB of delta -window, and further clones wait for one to finish; listing refs does not -wait. An `upload-pack` request is limited to 100MB, before and after gzip. - -Repositories are kept under `{cache_dir}/{name}-{hash of the URL}/`, so -repointing an upstream starts a fresh cache, and the old URL's directory is -removed at startup. Nothing else in `cache_dir` is touched, and the -directories of upstreams no longer configured are only logged, so remove -those yourself. The proxy locks `cache_dir` while it runs, through a -`.gitcache-lock` file in it, so a second process given the same directory, -such as another replica on a shared volume, fails to start; give each its own. - -Each repository's directory is named by its path as requested. Where -`cache_dir` ignores case in file names, as on macOS and Windows by default, -paths that differ only in case (`acme/widgets` and `acme/WIDGETS`) share one -directory and are treated as one repository. Use a case-sensitive file system -for an upstream whose repositories differ only in case. - -Git LFS objects are cached in the artifact cache, so they use the configured -storage backend and retention rather than `cache_dir`. git-lfs finds the LFS -API under the rewritten remote, at `/git/{name}/{repo}.git/info/lfs`, with no -further configuration. A batch request is passed to the upstream, which -decides which objects the repository has, and each download is pointed at -the proxy. An object the cache doesn't have is fetched from the download URL -the batch returned, which the proxy keeps in memory until shortly before it -expires, or else from one a fresh batch request returns. That URL is often -on another host than the Git server, such as GitHub's or Hugging Face's -object storage; the proxy fetches from whatever host the upstream names, -with the usual address checks, and `upstream.auth` credentials only go to -the hosts they are configured for. A download that isn't the object's size -is cut off. If the object's content is already stored under another -repository, such as a fork, it is copied from there instead of downloaded, -once the upstream has confirmed this repository has it; the copy takes its -own space. - -Large LFS objects may need longer timeouts. An object the proxy doesn't -have is sent to git-lfs as it is downloaded (see [Cache misses](#cache-misses)), -but the download must finish within `artifact_timeout` and the whole response -within `write_timeout`; both default to 5 minutes. -Download URLs are often signed; only the URL without its query is logged -and recorded with the object. An object is only cached once its content -matches its sha256 object ID. Objects are cached per repository, so one -repository's objects are never served for another. When the upstream can't -be reached or fails, a batch is answered with the objects cached for the -repository, and the others are reported as unavailable; a batch the upstream -refuses is refused. A cached object stays available after it is removed -upstream, until retention removes it or you delete it through the -[cache delete API](#cache-delete-api). - -Pushing, LFS uploads and locking, and the dumb HTTP protocol are not -supported. The settings -under `git` can also be set with `PROXY_GIT_CACHE_DIR`, `PROXY_GIT_MAX_SIZE`, -`PROXY_GIT_MAX_REPO_SIZE`, `PROXY_GIT_REFRESH_WAIT` and -`PROXY_GIT_STALE_MAX_AGE`. - -### Maven repositories - -```yaml -upstream: - - # The default repository, served at /maven/. + npm: "https://registry.npmjs.org" maven: "https://repo1.maven.org/maven2" - - # Additional Maven repositories, served at /maven/{name}/. - maven_repositories: - grails: "https://repo.grails.org/grails/restricted" + gradle_plugin_portal: "https://plugins.gradle.org/m2" + cargo: "https://index.crates.io" + cargo_download: "https://static.crates.io/crates" ``` -Builds often need more than one Maven repository. Each entry in -`upstream.maven_repositories` is served at `/maven/{name}/` with the upstream's -layout, e.g. `/maven/grails/org/grails/grails-core/6.0.0/grails-core-6.0.0.pom`, -and is fetched from that upstream alone: unlike the default repository, a named -one doesn't fall back to the Gradle Plugin Portal. Files are cached per -repository, so the same coordinates in two repositories never share a cache -entry. - -In every repository, a file the proxy would cache (an artifact, checksum or -signature) whose group directories contain a dot or colon -(`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, is not in -the Maven layout and gets 404 without an upstream request. Its group would read -the same as `org/foo`'s. Other paths there, such as `maven-metadata.xml` or a -directory listing, are handled as anywhere else. - -`upstream.maven_repositories` is additive. `/maven/…` keeps serving -`upstream.maven` with unchanged cache identities. A repository name is matched -before the default repository, so it hides every group ID whose first segment -is the same name there: a repository named `tools` would take -`/maven/tools/jackson/…` away from Maven Central. At config load the proxy -refuses names of two characters or less, common top-level domains such as `com` -and `io`, and every first segment of a group ID on Maven Central, the Gradle -Plugin Portal and Google's Maven repository (about 1,200 names, such as `javax`, -`junit`, `software` or `gradle`). Names are compared case-sensitively, as paths -are, and may only use letters, digits, `.`, `_` and `-`. Other repositories can -have group IDs the list doesn't know, so pick names that can't be the start of -a group ID, such as `grails` or `company-releases`. - -A repository's files are cached under its name and a hash of its upstream URL, -so pointing a repository at a different URL stops serving what the old one -sent. - -This field has no environment variable, as with the other named upstream maps. -`PROXY_UPSTREAM_MAVEN` still sets `upstream.maven`. - ## Authentication -Configure authentication for private upstream registries. The same authentication-aware client is used for metadata and artifact downloads, and credentials can reference environment variables using `${VAR_NAME}` syntax. - -OCI registries that return a Bearer challenge from a `/v2/{repository}/…` endpoint are handled automatically. The proxy discovers the token realm from `WWW-Authenticate`, applies any configured credentials for the token URL, and reuses the token until shortly before it expires. The token is always requested for pull access to the repository being fetched (`repository:{repository}:pull`), whatever scope the challenge names. - -A token fetched with credentials configured for the token URL is only fetched for, and sent to, a registry that is trusted with it: one on the same host and port as the token service, one configured with the same credentials as the token URL, or Docker Hub (`registry-1.docker.io` and `index.docker.io` with `auth.docker.io`) and GitLab.com (`registry.gitlab.com` with `gitlab.com`). Sharing a parent domain is not enough, so `team-a.jfrog.io` is not trusted with tokens from `team-b.jfrog.io`. Any other host that answers with a challenge naming that token service gets its 401 back, so it cannot obtain a token minted from your credentials. Tokens from a token service with no configured credentials are anonymous and are fetched for any registry, as before. If your registry and its token service are on different hosts, such as a self-hosted GitLab with its registry on its own hostname, configure the same credentials for both URLs: - -```yaml -upstream: - auth: - "https://registry.gitlab.example.com": - type: basic - username: "deploy-token" - password: "${GITLAB_DEPLOY_TOKEN}" - "https://gitlab.example.com/jwt/auth": - type: basic - username: "deploy-token" - password: "${GITLAB_DEPLOY_TOKEN}" -``` +Configure authentication for private upstream registries. Auth is matched by URL prefix, and credentials can reference environment variables using `${VAR_NAME}` syntax. ### Bearer Token @@ -1111,24 +170,9 @@ upstream: header_value: "${MAVEN_TOKEN}" ``` -### AWS ECR - -Private ECR registries issue authorization tokens that expire after 12 hours. The `ecr` auth type calls `ecr:GetAuthorizationToken` on demand, caches the result, and refreshes it shortly before expiry, so no static credential appears in the config file: - -```yaml -upstream: - oci: - ecr: "https://123456789012.dkr.ecr.eu-west-1.amazonaws.com" - auth: - "https://123456789012.dkr.ecr.eu-west-1.amazonaws.com": - type: ecr -``` - -AWS credentials are resolved by the SDK's default chain, which covers EKS IAM Roles for Service Accounts (IRSA), EC2/ECS instance profiles, `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` environment variables, and `~/.aws/credentials`. The IAM identity needs the `ecr:GetAuthorizationToken` action plus the usual `ecr:BatchGetImage` / `ecr:GetDownloadUrlForLayer` permissions on the target repositories. The region is inferred from private ECR IPv4, dual-stack, and FIPS hostnames. For other endpoint formats, set `region` explicitly or configure a default region for the SDK. - ### URL Matching -Auth keys must be absolute URLs. Matching compares the scheme, host, effective port, and path-segment prefix, preventing credentials for `registry.example.com` from being sent to a lookalike host such as `registry.example.com.evil.test`. The longest matching scope wins, so you can configure different credentials for different paths: +Auth configs are matched by URL prefix. The longest matching prefix wins, so you can configure different credentials for different paths: ```yaml upstream: @@ -1150,7 +194,7 @@ The `/gradle` endpoint supports optional safeguards for upload control and cache ```yaml gradle: build_cache: - read_only: true + read_only: false max_upload_size: "100MB" max_age: "168h" max_size: "20GB" @@ -1159,47 +203,14 @@ gradle: | Config | Environment | Description | |--------|-------------|-------------| -| `gradle.build_cache.read_only` | `PROXY_GRADLE_BUILD_CACHE_READ_ONLY` | Reject PUT uploads and serve GET/HEAD only (default `true`; the variable takes `true`/`false` or `1`/`0`, any other value keeps uploads off and logs a warning) | +| `gradle.build_cache.read_only` | `PROXY_GRADLE_BUILD_CACHE_READ_ONLY` | Disable PUT uploads and keep GET/HEAD read-only | | `gradle.build_cache.max_upload_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE` | Maximum accepted PUT body size (must be > 0) | -| `gradle.build_cache.max_age` | `PROXY_GRADLE_BUILD_CACHE_MAX_AGE` | Delete entries older than this duration (default `168h`, also accepts days such as `7d`; set `0` to disable) | +| `gradle.build_cache.max_age` | `PROXY_GRADLE_BUILD_CACHE_MAX_AGE` | Delete entries older than this duration (default `168h`, set `0` to disable) | | `gradle.build_cache.max_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_SIZE` | Total size cap for `_gradle/http-build-cache`, deleting oldest first (`0` disables) | | `gradle.build_cache.sweep_interval` | `PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL` | Frequency for background eviction sweeps | -Uploads are off by default, so a client that can reach the proxy cannot overwrite cache entries that end up in other people's builds. Set `read_only: false` (or `PROXY_GRADLE_BUILD_CACHE_READ_ONLY=false`) on the instance your CI jobs upload to, and let only CI push (`isPush = System.getenv("CI") != null`). Before this default changed, uploads were accepted without configuration; setups that upload need the setting now. - `max_age` and `max_size` are independent and can be combined. When both are set, age-based eviction runs first, then size-based eviction trims remaining entries oldest-first. -## Cooldown profiles - -Set `cooldown.profiles` to offer several policies from one instance. Clients select a profile through their package manager's registry URL: - -```yaml -cooldown: - default: "0" - profiles: - 3d: - default: "3d" - 7d: - default: "7d" - packages: - "pkg:npm/@example/tool": "0" -``` - -For the `3d` profile, use these paths after your proxy's base URL: - -| Client | Path | -|--------|------| -| npm registry | `/cooldown/3d/npm/` | -| Go `GOPROXY` | `/cooldown/3d/go` | -| Cargo registry | `sparse+https://your-proxy.example/cooldown/3d/cargo/` | -| Bundler RubyGems mirror | `/cooldown/3d/gem` | - -The prefix works with all package protocol routes. Cooldown has an effect only for [registries with cooldown support](../README.md#supported-registries). Each profile defines its own `default`, `ecosystems`, `packages` and `package_patterns`; it does not inherit the global cooldown overrides. Existing unprefixed URLs, the UI, APIs and background mirror jobs use the global policy. - -Profile names contain lowercase letters, digits, hyphens and underscores, and start with a letter or digit. Profiles must be configured before startup; an unknown profile returns 404. Nested profiles are rejected. Profile paths expose package protocols only, so `/cooldown/3d/ui/` and `/cooldown/3d/api/` return 404. - -Database records, artifacts and upstream metadata are shared. Generated download URLs retain the profile prefix, and filtered metadata and its validators are separated by profile. Changing a registry URL does not rewrite artifact URLs already recorded in a lockfile; regenerate those URLs if the selected policy must cover the whole installation. - ## Cooldown The cooldown feature hides package versions published too recently, giving the community time to spot malicious releases before they reach your projects. When a version is within its cooldown period, it's stripped from metadata responses so package managers won't install it. @@ -1213,8 +224,6 @@ cooldown: packages: "pkg:npm/lodash": "0" "pkg:npm/@babel/core": "14d" - package_patterns: - "pkg:npm/@example/*": "0" ``` | Config | Environment | Description | @@ -1222,299 +231,25 @@ cooldown: | `cooldown.default` | `PROXY_COOLDOWN_DEFAULT` | Global default cooldown | | `cooldown.ecosystems` | - | Per-ecosystem overrides | | `cooldown.packages` | - | Per-package overrides (keyed by PURL) | -| `cooldown.package_patterns` | - | Per-package glob overrides (keyed by PURL glob) | Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to disable. -Package PURL keys are normalized to canonical form before matching, so `pkg:npm/@babel/core` and `pkg:npm/%40babel/core` are equivalent, as are `pkg:pypi/Django` and `pkg:pypi/django`. If both forms configure the same package, the canonical entry wins. +Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages. -Go module names in exact cooldown overrides are matched case-insensitively. If both lowercase and mixed-case keys are configured, the lowercase entry wins. Go package patterns match lowercase module names. +Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata. -`package_patterns` matches canonical, versionless PURLs using `*` for zero or more characters and `?` for one character. Neither wildcard crosses `/` separators. Character classes (`[...]`) and backslash escapes are rejected at startup. For example, `"pkg:npm/@example/*"` matches packages under the `@example` npm scope. Patterns accept `@` as an alias for `%40`; other characters must use their canonical PURL form. Equivalent patterns with different durations are rejected at startup. Equal durations, such as `1d` and `24h`, are accepted. - -Exact `packages` entries take precedence over patterns. When several patterns match, longer patterns win after excluding `*` and `?` from the length. Ties use lexical order of the normalized patterns. - -Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. - -The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. - -Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. - -Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. - -Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` or Gradle `.module` itself is downloaded, which clients do before a version's other files, and otherwise with a `HEAD` request. A `.pom` or `.module` refused this way is still cached, so it counts toward `max_size` before the version leaves cooldown. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. - -Maven caveats: - -- A Nexus or Artifactory upstream may report when it cached the `.pom` rather than when it was published, so a version can stay in cooldown longer than it should. -- On Google's Maven Central mirror, `Last-Modified` is when the file reached the mirror's bucket, a few hours after it was published, so versions stay in cooldown a little longer. The upload time is in `x-goog-meta-last-modified`, which the proxy does not read. It is missing for files from the mirror's 2019 bulk import. -- Maven Central lists versions in version order, not in the order they were published. A patch to an older release line published after a newer release that is out of cooldown stays listed while it is in cooldown, but its files are still refused. -- SNAPSHOT versions are not held back. - -Helm uses each release's `created` time from the repository's `index.yaml`. Releases in cooldown are left out of the index the proxy serves, and their archives aren't served; a release without `created` is let through. Charts pulled over OCI have no publish time and aren't held back. - -Cargo, RubyGems, Hex, pub.dev, Conda and Helm check artifact downloads against cooldown, including files cached before the policy changed. When a publish-time lookup fails, an already-cached file is still served; an uncached download fails with 502, or 503 with `Retry-After` when the rate-limit gate holds the request. Conda checks timestamps per file because builds of one version can have different publish times. As in conda, a repodata `timestamp` is read as milliseconds when it is above 253402300799 (the end of year 9999 in seconds) and as seconds otherwise, since some older entries use seconds. - -Conda repodata and Helm indexes are parsed once per cached document for download checks. With `cache_metadata` off, the parsed index is kept for one minute, including the fetch, so an install can reuse it across files. Conda streams repodata when its compressed copy exceeds `metadata_max_size`, under the same decompressed size bound as the metadata filter. - -RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. - -For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, and the filtered result is kept until the first version it held back leaves cooldown (see [rewritten metadata cache](#rewritten-metadata-cache)); a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. - -With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead; other tags that name a version in cooldown are removed, as for the denylist, and `npm dist-tag ls` shows the same tags. A tarball request for a version the cached packument doesn't list yet fetches the packument again, within `metadata_ttl` or not, and is decided on what the upstream lists now; if the packument can't be fetched, the download is not let through, as when a publish time can't be read. - -In pub.dev package metadata, when `latest` is in cooldown it names the newest stable version out of cooldown instead, or the newest prerelease when no stable version is left, and it is left out when every version is in cooldown. - -Conda cooldown covers `repodata.json`, `current_repodata.json`, `repodata_from_packages.json` and sharded repodata (CEP 16). Other forms of the repodata, such as `repodata.json.zst`, `repodata.json.bz2` and `repodata.jlap` patches, would list versions still in cooldown, so with cooldown on they are answered 404 and conda and mamba fall back to `repodata.json`. With `cache_metadata`, `repodata.json` is read through the metadata cache as it is without cooldown: within `metadata_ttl` the stored copy is filtered as it is read from storage, after it the upstream is asked with its ETag, and while the upstream is down the stale copy is filtered. Once a document has been filtered, the proxy remembers the ETag of the result until the first entry it left out leaves cooldown, so a client revalidating with `If-None-Match` gets `304 Not Modified` without the document being filtered or sent again. The first response for each document has no ETag, as it is sent while it is filtered. Without `cache_metadata`, `repodata.json` is fetched and filtered on every request. Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. - -Hex cooldown requires `HEX_UNSAFE_REGISTRY=1` because filtering releases re-encodes the protobuf payload without the original signature. The proxy preserves the package name and repository fields, so origin verification can stay enabled. `HEX_NO_VERIFY_REPO_ORIGIN` controls origin verification and does not disable signature verification. Without a cooldown, leave signature verification enabled too. - -## Version denylist - -Block exact package versions independently of cooldown and artifact scanning: - -```yaml -denylist: - packages: - - "pkg:pypi/requests@2.31.0" - - "pkg:cargo/some-crate@1.2.3" - - "pkg:npm/%40scope/example@4.5.6" -``` - -Entries must be versioned PURLs without subpaths or qualifiers (except a conda -`channel`, below). Package names are canonicalized (including npm scopes); -versions match exactly, not by range or glob. Invalid entries prevent startup. -An empty list disables the policy. Restart the proxy after changing the list. - -Names and versions are compared the way each ecosystem treats them: - -- PyPI names ignore case, and runs of `-`, `_` and `.` are equivalent (PEP 503), - so `pkg:pypi/zope.interface@6.0` also covers `zope_interface-6.0-*.whl`. -- NuGet ids and versions ignore case. -- Debian, RPM, Alpine, Arch and FreeBSD entries ignore the distribution - namespace: `pkg:deb/debian/curl@8.5.0-2` and `pkg:deb/curl@8.5.0-2` are the - same entry. -- Debian and RPM filenames leave out the epoch, so an entry with one also - covers the version without it: `pkg:deb/foo@1:2.3-1` denies - `foo_2.3-1_amd64.deb`. An entry without an epoch only matches a version - without one. -- A RubyGems entry also covers that version's platform gems: - `pkg:gem/nokogiri@1.16.0` denies `nokogiri-1.16.0-x86_64-linux.gem`. -- Composer versions ignore a leading `v`, which Packagist keeps from the tag: - `pkg:composer/symfony/console@6.1.7` also covers `v6.1.7`. -- Go versions are matched as written, `pkg:golang/example.com/mod@v1.0.0-RC1`, - though download URLs escape uppercase letters (`v1.0.0-!r!c1`). - Go module names in denylist entries are matched case-insensitively. -- A Swift entry names the registry scope and package, ignoring case: - `pkg:swift/apple/swift-nio@2.62.0`. The stored form, - `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too, and so does - the source repository, as SBOM tools and GHSA advisories write it: - `pkg:swift/github.com/apple/swift-nio@2.62.0`. The proxy looks a source - repository up through `upstream.swift`'s `/identifiers` when `serve` or - `proxy mirror` starts and denies every identity listed for it; neither runs - when one can't be resolved. -- An OCI entry names the repository as it is pulled through the proxy and a - digest or tag: `pkg:oci/library/alpine@sha256:` refuses the blob or - manifest with that digest, including a manifest a tag resolves to, and - `pkg:oci/library/alpine@3.19` refuses pulling that tag. Repositories on a - named upstream keep their prefix, as in `pkg:oci/upstream/ghcr/org/image@...`. - An entry covers only the digest it names: denying an image index does not - deny the platform manifests and layers it lists to a client that requests - them by digest directly. -- A conda entry covers the package in every channel. Add a `channel` qualifier, - such as `pkg:conda/numpy@1.26.4?channel=conda-forge`, to deny it in one - channel only. - -Denied versions are removed from npm packuments, Cargo sparse indexes, and PyPI -simple HTML/JSON (files and the JSON `versions` list) and release metadata, including responses served from cached -metadata during an upstream outage. No publication timestamp or enabled cooldown -is required. npm tags pointing at denied versions are removed; `latest` is moved -to the highest remaining stable version when available. Other tags are not -retargeted. PyPI version-specific JSON endpoints return 404 for denied versions. - -The shared artifact download pipeline returns 403 for denied package/version -identities before reading storage, issuing a signed redirect, or fetching bytes. -This also applies to mirror downloads and recognized Debian package downloads. -Existing cached artifacts are retained, so removing a denial restores access. -Already-issued storage URLs and clients' local caches cannot be revoked by this -configuration. - -Metadata filtering currently covers npm, PyPI, and Cargo. Other ecosystems' -metadata is unchanged; in particular, APT indexes cannot be rewritten without -regenerating their checksums and signatures. Pass-through resources without a -resolved package/version identity are not covered. This is an explicit operator -policy, not automatic CVE detection or cache re-scanning. Exact pins or dependency -constraints may still make an install fail when no allowed version can satisfy it. - -### APT version pinning - -For Debian and Ubuntu clients, combine the proxy's download denylist with -client-side APT preferences. The proxy blocks recognized denied `.deb` downloads -with 403, but does not remove versions from APT indexes. Without a client-side -pin, APT can select a denied version and then fail when downloading it. - -On each client, add a record to `/etc/apt/preferences.d/proxy-denylist.pref`: - -```text -Package: somepkg -Pin: version 1.2.3-1 -Pin-Priority: -1 -``` - -Replace `somepkg` with the binary package name and `1.2.3-1` with its full APT -version, including the epoch if present. A negative priority excludes that -version from normal candidate selection. For multi-architecture clients, use -`Package: somepkg:any` to cover all architectures. Separate additional records -with blank lines. See [apt_preferences(5)](https://manpages.debian.org/stable/apt/apt_preferences.5.en.html). - -Check the effective priorities and simulate installation before deploying the -preferences broadly: - -```bash -apt-cache policy somepkg -apt-get --simulate install somepkg -``` - -Confirm that the denied version has priority `-1` and is not selected. Existing -specific pins can take precedence, so check for conflicts. APT can select -another candidate only if one is available and satisfies dependency constraints; -a Debian suite commonly offers only one version. Pinning does not remove an -already-installed package or automatically downgrade it. - -These preferences are managed on clients, not distributed by the proxy. Keep -the corresponding proxy denylist entries as a separate download guard. - -APT indexes are left unchanged to preserve upstream authentication: `Release` -metadata contains index checksums and is signed by the repository. Filtering -would require regenerating metadata and signing it with a key clients trust, -making the proxy a repository authority rather than a transparent cache. Do not -disable signature verification to work around this limitation. See -[apt-secure(8)](https://manpages.debian.org/stable/apt/apt-secure.8.en.html). - -If centrally curated APT indexes are required, use a repository manager such as -[aptly](https://www.aptly.info/) or [reprepro](https://salsa.debian.org/debian/reprepro) -as the upstream behind the proxy. Filtered APT repository generation and signing -are outside the proxy's current scope. - -## Artifact Scanning - -Cooldown only ever looks at a version's *publish timestamp* — it never inspects the actual bytes of an artifact. Artifact scanning runs after a fetched artifact is staged into storage but before it becomes visible from cache, so an external scanner (trivy, ClamAV, Wiz, or any custom service) can block a bad verdict from ever reaching a client. - -```yaml -scanning: - enabled: true - fail_open: false - timeout: 30s - signing_key: ${PROXY_SCANNING_SIGNING_KEY} - fetch_base_url: http://proxy.internal:8080 - scanners: - - name: clamav - url: http://clamav-adapter:8080/scan - mode: block - - name: trivy - url: http://trivy-adapter:8081/scan - mode: monitor - ecosystems: [npm, pypi] -``` - -| Config | Environment | Description | -|--------|-------------|-------------| -| `scanning.enabled` | `PROXY_SCANNING_ENABLED` | Turn on the scan gate. When false (default), artifacts are cached exactly as if scanning didn't exist | -| `scanning.fail_open` | `PROXY_SCANNING_FAIL_OPEN` | Treat scanner errors/timeouts as allow instead of block. Default is fail-closed | -| `scanning.timeout` | `PROXY_SCANNING_TIMEOUT` | Per-scan-call timeout, Go duration syntax (default `30s`) | -| `scanning.signing_key` | `PROXY_SCANNING_SIGNING_KEY` | Signs pull requests to the internal scan-fetch route. Required whenever `enabled` is true | -| `scanning.fetch_base_url` | `PROXY_SCANNING_FETCH_BASE_URL` | Address scanners use to reach this proxy to pull staged artifacts. Defaults to `base_url` | -| `scanning.scanners` | - | List of external scanning services (YAML only) | -| `scanning.scanners[].name` | - | Identifies this scanner in logs and metrics | -| `scanning.scanners[].url` | - | Endpoint the proxy POSTs scan notifications to | -| `scanning.scanners[].mode` | - | `block` (default) or `monitor` | -| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems. Valid names: `alpine`, `arch`, `cargo`, `chef`, `composer`, `conan`, `conda`, `cran`, `deb`, `freebsd`, `gem`, `generic`, `git` (Git LFS objects), `golang`, `helm`, `hex`, `julia`, `maven`, `npm`, `nuget`, `oci`, `pub`, `pypi`, `rpm`, `swift`. The aliases `go`, `docker`, `rubygems`, `packagist`, `debian` and `apk` are accepted too; any other name stops the proxy at startup | -| `scanning.scanners[].headers` | - | Extra HTTP headers sent with every scan request (e.g. for authenticating to the scanner service). Values support `${VAR_NAME}` expansion | - -### How caching defers to a scan verdict - -The proxy never uploads artifact bytes to a scanner. When an artifact is fetched from upstream, it's stored to the configured storage backend first, exactly as without scanning. If scanning is enabled for the artifact's ecosystem, the proxy then notifies each applicable scanner with package metadata and a short-lived, HMAC-signed URL pointing at the internal `/_internal/scan-fetch` route; each scanner GETs that URL itself to pull the exact bytes staged in storage and runs its own scan against them. With a `block`-mode scanner for the ecosystem, the client gets nothing before the verdict; with only `monitor`-mode scanners, a cache miss is sent to the client while it is stored, and the scan runs after (see [Cache misses](#cache-misses)). - -Scanners configured for the same ecosystem all run concurrently, never sequentially. The moment any `block`-mode scanner reports a not-allowed verdict (or errors, unless `fail_open` is set), the proxy cancels the in-flight calls to the other scanners and deletes the staged artifact — it's never committed to the cache database, so it was never visible to a client. If nothing blocks, the proxy waits for every `block`-mode scanner to finish before caching the artifact and serving it. A `monitor`-mode scanner's findings are logged and never gate the wait or the caching decision, even when it reports not-allowed. - -A blocked download surfaces to the client as `403 Forbidden` with the scanner's reason, across every ecosystem handler. - -Because scanners pull the staged bytes from the running server, the `proxy mirror` command cannot scan what it fetches. While scanning is enabled it refuses every package and container image instead of caching them unscanned. Mirror through the `/api/mirror` endpoints of the running server, whose jobs are scanned like any other download. - -### Scanner HTTP contract - -Any external service that implements this contract can act as a scanner — a trivy wrapper, a clamav-rest bridge, a Wiz connector, or an in-house service. The proxy POSTs a notify request to `scanning.scanners[].url` and waits for a JSON verdict. - -**Request** - -| Field | Type | Description | -|-------|------|-------------| -| `ecosystem` | string | e.g. `npm`, `pypi`, `cargo` | -| `name` | string | Package name | -| `version` | string | Package version | -| `filename` | string | Artifact filename | -| `purl` | string | Package URL (PURL) identifying this exact version | -| `content_type` | string | Artifact content type | -| `size` | integer | Artifact size in bytes | -| `fetch_url` | string | Short-lived signed URL; GET this to retrieve the exact staged bytes | - -```json -{ - "ecosystem": "npm", "name": "left-pad", "version": "1.0.0", - "filename": "left-pad-1.0.0.tgz", "purl": "pkg:npm/left-pad@1.0.0", - "content_type": "application/octet-stream", "size": 1234, - "fetch_url": "https://proxy.internal/_internal/scan-fetch?path=...&exp=...&sig=..." -} -``` - -**Response** - -| Field | Type | Description | -|-------|------|-------------| -| `allowed` | boolean | Whether the artifact may be cached and served | -| `reason` | string | Human-readable reason, surfaced to the client when `allowed` is false | -| `findings` | array | Optional list of `{"severity", "title", "description"}` objects | - -```json -{ - "allowed": false, - "reason": "malware detected", - "findings": [ - {"severity": "critical", "title": "Trojan.GenericKD", "description": "..."} - ] -} -``` - -The scanner must respond within `scanning.timeout` (default `30s`); a timeout is treated the same as a `block` verdict unless `fail_open` is set. - -### The `/_internal/scan-fetch` route - -`fetch_url` points at an internal route, `/_internal/scan-fetch`, that streams a staged object straight from the proxy's storage backend via a short-lived HMAC-signed token (`path`, `exp`, `sig` query parameters). This works identically across every storage backend — local filesystem, S3, GCS, Azure — since it never depends on a backend-specific presigned URL, only on the one storage operation every backend already implements. - -This route is not part of the public API. It's meant only for scanners to pull artifacts they've been notified about, and should be restricted to internal-network access at the ingress/network-policy layer — the HMAC scoping (one object, a short TTL) limits what a leaked token can do, but isn't a substitute for network restriction. Its query parameters are also documented in the generated [OpenAPI spec](../README.md#openapi-swagger). - -The route only exists when scanning is actually configured: it's not mounted at all unless at least one scanner is enabled and `scanning.signing_key` is set, and it also refuses every request with `404` if either condition somehow isn't met at request time. There is no way to reach it, even with a forged token, when scanning is disabled. +Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. ## Metadata Caching By default the proxy fetches metadata fresh from upstream on every request. Enable `cache_metadata` to store metadata responses in the database and storage backend for offline fallback. When upstream is unreachable, the proxy serves the last cached copy. ETag-based revalidation avoids re-downloading unchanged metadata. -OCI manifests and tag lists are always cached because cached image blobs cannot be pulled without their manifests and offline clients may need tag resolution. Digest-addressed manifests are immutable and served directly from cache. Tag-addressed manifests and tag lists follow `metadata_ttl`, revalidate when stale, and fall back to the last cached response when the registry is unavailable. - -A stale tag is revalidated with a HEAD request carrying the client's `Accept` header, and the `Docker-Content-Digest` it returns is compared with the cached one. If the tag did not move, or moved to a manifest already cached by digest, nothing else is fetched. Otherwise the new manifest is fetched once by digest and stored under the tag and the digest. Docker Hub counts manifest GETs, but not HEADs, against its pull rate limit, so an unchanged tag costs no pulls however often its TTL runs out. A registry whose HEAD response has no digest gets a conditional GET for the tag instead. A client's HEAD request for a stale or uncached tag is passed on as a HEAD; when the digest it returns is cached, the tag is pointed at it, so the GET that follows is served from cache. Concurrent requests for the same manifest share one upstream request, and instances sharing a Postgres database take turns through the same [fetch lease](#several-instances) as other metadata. - ```yaml cache_metadata: true ``` Or via environment variable: `PROXY_CACHE_METADATA=true`. -Some metadata never changes once published, and is cached like an artifact, with or without `cache_metadata`, and served without revalidation whatever `metadata_ttl` says: a Go module's `.mod` and `.info` for a canonical version (a release, pre-release, pseudo-version or `+incompatible` version; the checksum database fixes their content), and RPM repodata files named after their own checksum (`repodata/-primary.xml.gz`). A stored copy is checked against the digest recorded for it and fetched again if it doesn't match. Go branch and commit queries, `@v/list` and `@latest`, and RPM `repomd.xml` and plainly named repodata files follow `metadata_ttl` as before. A checksum-named repodata file over `metadata_max_size` is streamed from upstream uncached. - -Composer archive downloads look up their upstream URL in the same cached metadata, so an archive is resolved without asking Packagist again and stays installable while it is down. Only a version or file the cached metadata does not list yet is looked up upstream. With or without `cache_metadata`, a cached archive of a tagged release from the default repository is served without any metadata lookup; with cooldown enabled it is checked against the publish time recorded when the archive was first resolved. Branch (dev) versions, routed packages and archives picked by a query string such as `?sha=` are always resolved from the metadata. - -Swift release lists and manifests are cached with the upstream's `Link` header, which carries the release list's pages and the version-specific manifests a client chooses from. - The `proxy mirror` command always enables metadata caching regardless of this setting. ### Metadata TTL @@ -1531,60 +266,15 @@ Set to `"0"` to always revalidate with upstream (ETag-based conditional requests When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. -### Change feeds - -Within `metadata_ttl` a new release isn't seen. For npm and Composer the proxy can instead follow the registry's change feed, which lists the packages that changed, and revalidate just those: - -```yaml -cache_metadata: true -upstream: - npm_change_feed: true - composer_change_feed: true -``` - -Or via environment variables: `PROXY_UPSTREAM_NPM_CHANGE_FEED=true`, `PROXY_UPSTREAM_COMPOSER_CHANGE_FEED=true`. Both require `cache_metadata`, and do nothing with a `metadata_ttl` of `"0"`. - -Every minute the proxy reads what changed since its last read. Cached metadata of a listed package that was fetched before the read is marked as needing revalidation, and the next request for it asks the upstream with its `ETag` (a `304` when nothing changed). Artifacts are not touched. Listed packages the proxy hasn't cached cost nothing beyond the read. The feed is read in full, so the upstream doesn't learn which packages the proxy has cached. Packages in `upstream.npm_routes` or `upstream.composer_routes` are cached under other keys and never matched. - -- Composer reads `{upstream.composer}/metadata/changes.json`, which packagist.org advertises as `metadata-changes-url`. Both `vendor/package` and its `vendor/package~dev` file are covered. Packagist keeps 24 hours of changes; if the proxy's position is older than that, Packagist asks for a resync and all cached Composer metadata is revalidated. repo.packagist.org sends `Cache-Control: max-age=900`, but Packagist purges its CDN when a package changes, so revalidation gets the new file within seconds; the proxy doesn't apply that header either way. -- npm reads `https://replicate.npmjs.com/registry/_changes`, the public registry's feed, whatever `upstream.npm` is. It is only useful when `upstream.npm` is registry.npmjs.org or serves the same packages. Packuments (`/npm/{name}`) are covered; version and dist-tag documents (`/npm/{name}/latest`) still wait out `metadata_ttl`. A busy minute on npm lists a few thousand packages; the proxy reads up to 100,000 per pass and continues where it stopped on the next. - -A new release therefore shows up about a minute after the registry lists it rather than up to `metadata_ttl` later. The feed position is kept in memory: after a restart the proxy starts from the feed's current position, and changes made while it was down are covered by `metadata_ttl` as before. While a feed can't be read the proxy logs a warning and retries with a growing delay, up to 15 minutes, and `metadata_ttl` applies meanwhile. A request whose upstream fetch overlaps the change and the feed read can still store the old document for one TTL. Each instance sharing a database reads the feeds itself. `proxy_metadata_expired_by_change_feed_total` counts the cached documents marked, by ecosystem, and debug logging names each one. - ### Metadata size limit -Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. With `cache_metadata` and the metadata rewrite cache on, an npm packument is streamed from the upstream into storage and rewritten from there in two passes, so it isn't held in memory and the limit doesn't apply to it; the rewritten copy is kept gzip-compressed. Requests for a single version (`//`) and tarball downloads read the version from the stored copy the same way. `metadata_stream_max_size` (1 GB by default) caps a document streamed this way; one over it isn't stored, and the stored copy, if any, is served instead. +Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. ```yaml -metadata_max_size: "100MB" # default -metadata_stream_max_size: "1GB" # default +metadata_max_size: "100MB" # default ``` -Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`, and `PROXY_METADATA_STREAM_MAX_SIZE=2GB` for `metadata_stream_max_size`. - -### Artifact size limit - -Artifacts have no size limit by default. `artifact_max_size` refuses any artifact larger than the limit, so a client cannot fill storage or tie up the proxy by requesting huge downloads. An artifact whose upstream `Content-Length` is over the limit is refused before its body is read; one without a declared size fails once the download passes the limit. Either way nothing is cached and the client gets `502 Bad Gateway`, except when artifacts are streamed: one without a declared size has already started sending when it passes the limit, so the client sees the download cut off. The limit applies to `serve`, `mirror` and streamed artifacts. - -```yaml -artifact_max_size: "2GB" # default: unlimited ("" or "0") -``` - -Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`. - -### Rewritten metadata cache - -The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite expires when the first version it held back leaves cooldown, since versions it let through stay allowed and a new upstream version changes the document. A rewrite that held nothing back is kept for the shortest cooldown configured, at least a minute and at most a day. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. - -Each cached document is kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it, as npm, pnpm, Yarn and Composer do. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. Only the compressed copy is kept, typically an eighth of the document, and it is decompressed for the clients that don't accept gzip. Documents rewritten outside the cache are served uncompressed. - -Size the cache for the metadata of the packages one install asks for. pnpm 12 checks every lockfile entry against the registry on each install unless `trustLockfile` is set, so each install reads the packument of every package in the lockfile; when they don't all fit, each install evicts what the next one needs and rewrites them all again. A lockfile of about 550 npm packages takes about 50 MB. - -```yaml -metadata_rewrite_cache_size: "256MB" # default; "0" rewrites on every request -``` - -Or via environment variable: `PROXY_METADATA_REWRITE_CACHE_SIZE=1GB`. +Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`. ## Upstream HTTP timeout @@ -1596,205 +286,7 @@ http_timeout: "30s" # default Or via environment variable: `PROXY_HTTP_TIMEOUT=2m`. -Set to `"0"` to disable the timeout entirely (requests then rely only on the server's write timeout). Independently of this setting, the shared transport gives up on an upstream that has not sent response headers within 60 seconds. - -## Hosted Packages - -Operators can push a package archive into the proxy under a name and version. The proxy serves it in place of the upstream archive with the same version, or in addition to the upstream's versions. Use it to ship a patched build of a dependency without running a separate registry. Files that aren't packages, such as prebuilt binaries other builds download, can be pushed as [generic files](#generic-files). - -The `/api/hosted` endpoints are disabled by default. Because a pushed version replaces the upstream one for every client, enabling them requires a token: - -```yaml -hosted: - enabled: true - token: "${PROXY_HOSTED_TOKEN}" - max_upload_size: "100MB" - npm_scopes: ["@acme"] - composer_vendors: ["acme"] -``` - -| Config | Environment | Description | -|--------|-------------|-------------| -| `hosted.enabled` | `PROXY_HOSTED_ENABLED` | Register the `/api/hosted` endpoints (default `false`) | -| `hosted.token` | `PROXY_HOSTED_TOKEN` | Bearer token required to push and delete; required when enabled, supports `${VAR}` | -| `hosted.max_upload_size` | `PROXY_HOSTED_MAX_UPLOAD_SIZE` | Maximum size of one pushed archive or file (default `100MB`) | -| `hosted.npm_scopes` | `PROXY_HOSTED_NPM_SCOPES` | npm scopes served from pushed packages only (comma-separated in the environment) | -| `hosted.composer_vendors` | `PROXY_HOSTED_COMPOSER_VENDORS` | Composer vendors served from pushed packages only | - -`hosted.enabled` cannot be combined with `storage.cache_artifacts: false`. - -With `hosted.enabled` off, archives pushed earlier stay in storage but are not served as hosted packages: package metadata doesn't list them, `/generic/hosted/` isn't served, and npm and Composer downloads take the normal proxy path, cooldown included. An upstream copy still never replaces a pushed archive. - -### Pushing a patched package - -1. Build the archive with the original package name and the version you push in its manifest (`package.json` or `composer.json`), e.g. `request-2.88.3.tgz`. -2. Push it: - - ```bash - export PROXY_HOSTED_TOKEN=... - proxy hosted push -url https://proxy.example.com npm request 2.88.3 ./request-2.88.3.tgz - ``` - - or with curl: - - ```bash - curl -X PUT --data-binary @request-2.88.3.tgz \ - -H "Authorization: Bearer $PROXY_HOSTED_TOKEN" \ - "https://proxy.example.com/api/hosted/npm/request/2.88.3?sha256=$(sha256sum request-2.88.3.tgz | cut -d' ' -f1)" - ``` - - The `sha256` query parameter is optional. When it is set, a push whose bytes don't match is refused with 400. `proxy hosted push` always sends it. - -3. Install it as usual, e.g. `npm install request@2.88.3`. The version is listed in the package metadata, so ranges like `^2.88.0` resolve to it too. -4. Check what is pushed: `proxy hosted list npm request` or `GET /api/hosted/npm/request`. -5. Remove it again: `proxy hosted delete npm request 2.88.3` or `DELETE /api/hosted/npm/request/2.88.3`. If the upstream has the version, clients get the upstream archive again. - -A pushed file is never replaced: pushing the same version again answers 409, also under a differently cased Composer name, before the body is uploaded when the client sends `Expect: 100-continue` (as `proxy hosted push` does). Clients that pinned the file by its hash would otherwise get different bytes under the same version. To push it again, delete it first. - -### Supported ecosystems - -Download handlers look archives up by the filename clients request, so a push is stored under that name: - -| Ecosystem | Name | Filename | -|-----------|------|----------| -| `npm` | `{name}` or `@{scope}/{name}` | `{name}-{version}.tgz`, without the scope | -| `composer` | `{vendor}/{package}` | `{package}-{version}.zip` | -| `generic` | `{name}`, one path segment | named in the push path, see [Generic files](#generic-files) | - -Other ecosystems are not supported yet. - -npm and Composer archives must contain the package manifest: `package.json` in the top directory of an npm tarball, or `composer.json` at the root or in the single top directory of a Composer zip. Its name, and its version when it has one, must match the push. Otherwise the push is refused with 400. - -### Package metadata - -Pushed versions are listed in the package metadata clients resolve against, with the dependencies from their manifest: - -- A pushed version takes the place of an upstream version with the same number. -- Pushed versions are added next to the upstream ones and are never held back by cooldown. npm `dist-tags` stay as the upstream has them, also in `npm dist-tag ls`. -- A package the upstream doesn't know is served from its pushed versions alone. Its npm `latest` tag is the highest pushed version. -- Pushed Composer dev versions (`dev-*`, `*-dev`) are listed in the `~dev` metadata file. -- Pushed Composer versions carry an empty `notification-url`, as routed packages do, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. A pushed `composer.json` that sets its own `notification-url` keeps it, and Composer then reports installs there. -- npm entries carry the archive's sha512 as `dist.integrity`. Composer entries carry its sha256 as `dist.reference` and leave `shasum` empty. - -Metadata of packages with pushed versions is rewritten on every request instead of being cached, so a push or delete shows up on the next request. - -### Hosted-only namespaces - -A package the upstream doesn't know is served from its pushed versions, but the proxy still asks the upstream about it on every request. If someone later publishes the same name there, their versions are listed next to yours, and a range such as `^1.0.0` can resolve to one of them (dependency confusion). - -For private packages: - -1. Prefer a namespace you own on the public registry: an npm scope registered on npmjs.com, or a Composer vendor with packages on Packagist. Nobody else can publish under it. -2. List the namespace in `hosted.npm_scopes` or `hosted.composer_vendors`. Its packages are served from pushed versions alone: the upstream is never asked about them, and anything not pushed answers 404. - -Other packages, including ones in other scopes and vendors, are proxied as usual. - -### Generic files - -Files that aren't packages, such as a prebuilt binary that other builds download and check against a pinned sha256, are pushed to the `generic` ecosystem. They have no manifest; the name, version and filename come from the path, and a version can hold several files: - -```bash -proxy hosted push generic tools 1.2.0 ./tool-linux-arm64.tar.gz -# or -curl -X PUT --data-binary @tool-linux-arm64.tar.gz \ - -H "Authorization: Bearer $PROXY_HOSTED_TOKEN" \ - https://proxy.example.com/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz -``` - -They are downloaded without a token from `/generic/hosted/{name}/{version}/{file}`, with the file's sha256 as `ETag`: - -```bash -curl -O https://proxy.example.com/generic/hosted/tools/1.2.0/tool-linux-arm64.tar.gz -``` - -Generic files never come from an upstream: anything not pushed answers 404. While hosting is enabled, `hosted` can't be used as an `upstream.generic` name. `proxy hosted list generic tools` lists all versions and files. `proxy hosted delete generic tools 1.2.0` removes a version with all its files, and `proxy hosted delete generic tools 1.2.0 tool-linux-arm64.tar.gz` removes only that file. - -### Storage and eviction - -Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones. - -## Project inventory - -The project inventory answers "which repository uses which version of a package?". Each repository uploads an SBOM of its default branch, and the proxy records the package versions it lists. The UI then shows: - -- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version. -- On a package page: which projects use each version, including versions the proxy hasn't cached. - -It is disabled by default. Uploads and deletes require a token: - -```yaml -projects: - enabled: true - token: "${PROXY_PROJECTS_TOKEN}" - max_age: "90d" -``` - -| Config | Environment | Description | -|--------|-------------|-------------| -| `projects.enabled` | `PROXY_PROJECTS_ENABLED` | Register the `/api/projects` endpoints and the UI pages (default `false`) | -| `projects.token` | `PROXY_PROJECTS_TOKEN` | Bearer token required to upload and delete; required when enabled, supports `${VAR}` | -| `projects.max_age` | `PROXY_PROJECTS_MAX_AGE` | Remove projects that haven't uploaded for this long, e.g. `90d` or `720h`; empty or `0` keeps them (default) | - -Removing old projects runs hourly. With several instances on one database, one of them does it. - -### Uploading an SBOM - -`PUT /api/projects/{project}/sbom` takes a CycloneDX (JSON or XML) or SPDX (JSON or tag-value) document of up to 32 MiB and replaces everything recorded for the project with what it lists. The optional `ref` and `branch` query parameters are stored and shown with the project. - -The project name is a path such as `group/sub/repo`: segments of letters, digits, `.`, `_` and `-`, at most 255 characters. It can be sent as path segments (`/api/projects/group/sub/repo/sbom`) or URL-encoded as one segment (`/api/projects/group%2Fsub%2Frepo/sbom`). - -```bash -syft dir:. -o cyclonedx-json > sbom.json -curl -X PUT --data-binary @sbom.json \ - -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ - "https://proxy.example.com/api/projects/acme/shop/web/sbom?ref=$(git rev-parse HEAD)&branch=main" -``` - -The response counts the recorded package versions and the skipped entries: - -```json -{"project":"acme/shop/web","components":9,"skipped":6} -``` - -Entries are recorded by their package URL, with the package named the way the proxy caches it: NuGet IDs lowercased, PyPI names normalized as in PEP 503 (`zope.interface` becomes `zope-interface`), the distribution dropped from `deb`, `rpm` and `apk` package URLs. Entries without a package URL (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`), entries without a version, and conda entries without a `channel` qualifier (the proxy records conda packages as `channel/name`) are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once. - -A GitLab CI job on the default branch could look like this: - -```yaml -sbom: - image: alpine:3.20 - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - script: - - apk add --no-cache curl - - curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin - - syft dir:. -o cyclonedx-json > sbom.json - - > - curl -sSf -X PUT --data-binary @sbom.json - -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" - "https://proxy.example.com/api/projects/$CI_PROJECT_PATH/sbom?ref=$CI_COMMIT_SHA&branch=$CI_COMMIT_BRANCH" -``` - -### Querying - -```bash -curl https://proxy.example.com/api/projects # all projects -curl https://proxy.example.com/api/projects/acme/shop/web # one project's package versions -curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash' # projects using lodash -curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash@4.17.20' # projects using lodash 4.17.20 -curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ - https://proxy.example.com/api/projects/acme/shop/web -``` - -Reading needs no token, like the rest of `/api`. With `?purl=`, each project carries the `version` it uses and no `component_count`. A delete names the project itself: `DELETE /api/projects/acme/shop/web/sbom` is answered with 400 unless a project `acme/shop/web/sbom` exists. - -A project's page in the UI also has a delete button. It asks for the token and sends it to `DELETE /ui/api/projects/{project}`, which works like the API endpoint and also requires the token; the token is kept for the browser session. Like `/ui/api/cache`, it sits under `/ui`, so rules for the UI at an ingress cover it. - -### What it shows - -The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. Versions are compared as the SBOM gives them: a version spelled differently from how the proxy caches it, such as Composer's `v6.4.1` for `6.4.1` or a Debian version with or without its epoch, shows as "not cached" on the package page. - -Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy. +Set to `"0"` to disable the timeout entirely (requests then rely only on the server's write timeout). ## Mirror API @@ -1808,82 +300,15 @@ Or via environment variable: `PROXY_MIRROR_API=true`. When disabled, the endpoints are not registered and return 404. -Start a mirror job with either PURLs or an inline CycloneDX or SPDX JSON document. Requests without `Content-Type: application/json` get 415, so other websites cannot start jobs through a visitor's browser: - -```bash -curl -X POST http://localhost:8080/api/mirror \ - -H "Content-Type: application/json" \ - -d '{"sbom":{"bomFormat":"CycloneDX","components":[{"purl":"pkg:npm/lodash@4.17.21"}]}}' -``` - -`platforms` limits which platforms of container images are mirrored, like the command's `--platform` flag: - -```bash -curl -X POST http://localhost:8080/api/mirror \ - -H "Content-Type: application/json" \ - -d '{"purls":["pkg:docker/library/alpine@3.20"],"platforms":["linux/amd64"]}' -``` - -`latest` mirrors only the newest version of PURLs without a version, like the command's `--latest` flag: - -```bash -curl -X POST http://localhost:8080/api/mirror \ - -H "Content-Type: application/json" \ - -d '{"purls":["pkg:npm/lodash"],"latest":true}' -``` - -## Cache Delete API - -Deleting cached artifacts is disabled by default. Enable it to register `DELETE /api/cache` and show Delete buttons on the version page of the web UI: - -```yaml -cache_delete: true -``` - -Or via environment variable: `PROXY_CACHE_DELETE=true`. - -The endpoint has no authentication. Before enabling it, restrict `/api/cache` and `/ui` to trusted users at your ingress or reverse proxy. The UI sends its deletes to `/ui/api/cache`, so whatever protects `/ui` protects those too. - -Delete every cached file of a version, or only the one named by `filename`: - -```bash -curl -X DELETE http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre -curl -X DELETE 'http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre?filename=guava-32.1.3-jre.jar' -``` - -The name is the package name the UI shows, for example `group:artifact` for Maven. An RPM package's `filename` is its path under `upstream.rpm` (`updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm`), since repositories under one upstream can ship different files under one name. The response lists the deleted files. The next request for a deleted file fetches it from the upstream again. The stored bytes are removed after the same delay as a replaced artifact, so downloads already in progress can finish. - -Pushed archives are not touched: a whole-version delete skips them, and naming one answers 409. Delete those through the [hosted API](#hosted-packages). - ## Mirror Command The `proxy mirror` command pre-populates the cache from various sources. It accepts the same storage and database flags as `serve`. -Packages are downloaded from the configured upstreams: npm, cargo, gem, golang, hex, pub, maven and nuget from `upstream.npm`, `upstream.cargo_download`, `upstream.gem`, `upstream.go`, `upstream.hex`, `upstream.pub`, `upstream.maven` and `upstream.nuget`. npm packages in `upstream.npm_routes` come from their route and are cached under the route's keys, so clients get them from the cache as they would after an install. Mirror API jobs do the same. - -A PyPI version is mirrored with every file `upstream.pypi` lists for it: the sdist and each wheel, checked against the listed sha256 and cached under the name and version the download handler reads from the filename. Add the `file_name` qualifier to mirror one file, for example `pkg:pypi/zope.interface@7.2?file_name=zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl`. - -A conda version is mirrored with every build anaconda.org lists for it, across all subdirs, and each build is cached the way `/conda/{channel}/{subdir}/{file}` serves it. Give the channel as the namespace or the `channel` qualifier, for example `pkg:conda/conda-forge/numpy@2.1.0`; a package without one is refused. The `subdir`, `build` and `type` qualifiers select builds, for example `pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&type=conda`. Builds with a listed sha256 are checked against it. Files are listed from anaconda.org, so mirroring conda fails while `upstream.conda` points somewhere else. - -A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`. - -A Helm chart is mirrored from a repository in `upstream.helm`, named as the namespace, for example `pkg:helm/prometheus-community/prometheus@27.1.0` with `prometheus-community` configured. The chart is fetched from the first URL the repository's `index.yaml` lists for that version, checked against the listed digest, and cached under the repository and digest `/helm/{name}/charts/...` serves it from. The index is cached as well, so with [`cache_metadata`](#metadata-caching) on `helm pull` works while the repository is down. Without a version every version in the index is mirrored. Charts named without a repository, or from one that isn't configured, are refused, and so are charts the index lists in an OCI registry: helm pulls those through `/v2`, so mirror them as `pkg:oci` images. - -A Julia package is mirrored with the source tarball `upstream.julia` serves at `/package/{uuid}/{tree-hash}`, cached under the package's name and tree hash as `/julia/package/...` serves it. The PURL needs the `uuid` qualifier, for example `pkg:julia/Example@0.5.5?uuid=7876af07-990d-54b4-ab0e-23690620f79a`. Only packages of the General registry can be mirrored: the uuid and tree hash are read from General on GitHub, also with another `upstream.julia`, since they are the same on every Pkg server serving General. The tarball isn't checked by the proxy, as when serving; Pkg checks the tree hash of what it unpacks, except on Windows. Dependencies, the artifacts in a package's `Artifacts.toml` and the registry aren't mirrored. An offline `Pkg.add` also needs the registry: one installed on the client, or the one the proxy cached when a client last fetched it, which it serves offline with `cache_metadata` on. - -A Swift release is mirrored from `upstream.swift` with its source archive, checked against the SHA-256 checksum in the release metadata and cached under the key `/swift/{scope}/{name}/{version}.zip` serves. Give the registry scope as the namespace, for example `pkg:swift/apple/swift-argument-parser@1.5.0` for `apple.swift-argument-parser`, or the source repository, as SBOM tools write it: `pkg:swift/github.com/apple/swift-argument-parser@1.5.0` is looked up through the upstream's `/identifiers`, and the first identity it lists is mirrored. The release metadata, the package's release list and the release's `Package.swift` with its version-specific manifests are stored in the metadata cache too, so `swift package resolve` works with the upstream unreachable. Serving them, and the archive, offline needs `cache_metadata` on the server, since the archive is served with the checksum from the release metadata. - -A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. - -An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. - | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | | `--concurrency` | `4` | Number of parallel downloads | | `--dry-run` | `false` | Show what would be mirrored without downloading | -| `--platform` | | Comma-separated `os/arch[/variant]` list limiting which platforms of container images are mirrored | -| `--latest` | `false` | Mirror only the newest version of PURLs without a version | | `--config` | | Path to configuration file | | `--storage-url` | | Storage URL | | `--database-driver` | | Database driver | @@ -1896,92 +321,6 @@ Positional arguments are treated as PURLs: proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 ``` -### Unversioned PURLs - -A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran, brew and swift. For cargo and conda the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. Julia versions are always listed from the General registry. - -With `--latest`, or `"latest": true` in a mirror API job, only the latest version is mirrored, as git-pkgs/registries picks it: the version the registry advertises as latest where it has one, such as npm's `dist-tags.latest`, and otherwise the newest by publish time, leaving out yanked, retracted and deprecated versions. PURLs with a version, including those in an SBOM, are mirrored as given. Run it from cron to keep the newest releases of a list of packages cached: - -```bash -proxy mirror --latest pkg:npm/lodash pkg:pypi/requests pkg:cargo/serde -``` - -### Container images - -`pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable. - -```bash -proxy mirror --platform linux/amd64,linux/arm64 pkg:docker/library/alpine@3.20 -proxy mirror 'pkg:docker/containerd/alpine@3.14.0?repository_url=ghcr.io' -proxy mirror 'pkg:oci/debian@sha256:...?repository_url=docker.io/library/debian&tag=bookworm' -``` - -- The version is a tag or a digest. Unversioned image PURLs fail; mirroring all tags is not supported. -- Without `repository_url` the name is routed like a pull without containerd's `ns` parameter: the default OCI registry, repository prefix routes and `upstream/{name}/`. Single-segment Docker Hub names get the `library/` prefix. -- `repository_url` is resolved like containerd's `ns` parameter, so it has to name the default registry, Docker Hub, a configured `upstream.oci` host or a host listed in `upstream.oci_mirrors`. Other hosts are refused. -- `pkg:oci` needs `repository_url`. With `tag=` the tag is mirrored as well; if it now points to another digest, both are mirrored and a warning is logged. `arch=` limits the platforms when `--platform` is not given. -- Without `--platform` every platform of an index is mirrored, including Windows images, which can be large. Attestation manifests of the selected platforms are mirrored too, because Docker fetches them on every pull. -- Non-distributable (foreign) layers are skipped. -- While `scanning.enabled` is set, the command refuses images, because scanners fetch staged layers from the running server. Use the mirror API instead. - -### Homebrew bottles - -`pkg:brew` PURLs mirror a homebrew/core formula's bottles, which Homebrew publishes as an image on `upstream.homebrew_artifact`. They are mirrored like that image, under the keys brew reads through `HOMEBREW_ARTIFACT_DOMAIN`, so `brew install` is served from cache, also when the upstream is unreachable: - -```bash -proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403 -``` - -- A formula name with `@` is written with `%40`, since `@` starts the version: `pkg:brew/openssl%403` is `openssl@3`, and `pkg:brew/openssl@3` is version 3 of `openssl`. -- The formula's JSON API on `upstream.homebrew_api` gives the image tag for the current version, which carries the formula's revision and the bottles' rebuild (`hello` 2.12.3 is tagged `2.12.3-1`). Only the current version is described there; another version is used as the tag as given, so give the full tag, such as `2.12.2` or `2.12.3-1`, for older bottles. -- A PURL without a version mirrors the current version. -- `--platform` selects bottles like image platforms: `linux/arm64`, `linux/amd64`, or `darwin/arm64` for every macOS release's arm64 bottle. Without it every bottle of the version is mirrored. -- Dependencies aren't mirrored with a formula; list them too. brew also needs the JSON API, which the proxy keeps with `cache_metadata`. -- Casks, source archives and taps other than homebrew/core aren't mirrored. - -### Git repositories - -`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID or a tag: - -```bash -proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 -proxy mirror pkg:git/github.com/git-pkgs/purl@v0.2.1 -proxy mirror pkg:git/github.com/git-pkgs/purl -``` - -- The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. -- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. -- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. When the upstream can't be reached, a cached tag is mirrored at the commit it last pointed to, with a warning in the log. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID. -- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`. -- The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. - -### Scheduled refresh - -Tags such as `alpine:3.20` move. The server can re-mirror them on its own, so the cache follows the tag even when nobody pulls it or the proxy is offline later: - -```yaml -mirror: - schedules: - - images: [pkg:docker/library/alpine@3.20, pkg:docker/library/nginx@stable] - interval: 6h - platforms: [linux/amd64] -``` - -| Setting | Env | Description | -|---------|-----|-------------| -| `mirror.schedules[].images` | - | `pkg:docker` or `pkg:oci` PURLs with a tag, resolved like the `mirror` command | -| `mirror.schedules[].interval` | - | How often the images are mirrored, at least `1m` | -| `mirror.schedules[].platforms` | - | `os/arch[/variant]` entries limiting the platforms; empty mirrors all | - -Each run checks the tag with a HEAD request. If the tag still points to the cached digest, nothing else is fetched. If it moved, the new manifests and blobs are fetched first and the tag is switched only when all of them are cached, so a failed run leaves the previous image pullable. Images of one schedule are mirrored one after the other. - -- The first run starts within ten minutes of startup, later runs every `interval` ±10 %, so restarts and several instances don't hit the registry at once. Instances sharing a PostgreSQL database run each schedule once per interval: the first to reach it records the run, and the others skip it. A run cut off by a shutdown is not recorded, so the next instance to reach the schedule runs it. -- Digest-pinned PURLs never move and are rejected. A `pkg:oci` digest with `tag=` is accepted; the tag is refreshed and a warning is logged when it no longer matches the digest. -- The previous digest stays cached, so pulls by digest keep working, until eviction or retention removes it. -- Docker Hub does not count HEAD requests against its pull limit, but every changed manifest is a pull. Set `platforms` and `upstream.auth` for `registry-1.docker.io` to stay under the anonymous limit. -- Mirrored layers are scanned when `scanning.enabled` is set. A blocked layer fails the run and keeps the previous image. -- Schedules are read at startup; changes need a restart. They do not need `mirror_api`. - ## Docker ### SQLite with Local Storage diff --git a/docs/docker-builds.md b/docs/docker-builds.md deleted file mode 100644 index 8509b9d..0000000 --- a/docs/docker-builds.md +++ /dev/null @@ -1,168 +0,0 @@ -# Using proxy in Docker builds - -Docker builds can share proxy's package cache even when their builders and local -package caches are new. Configure the package manager inside the build to use -proxy. Cached archives still need to be downloaded from proxy and installed; -native extensions still need to be compiled. - -## Run a persistent proxy - -Run proxy outside the image build and keep its database and artifact storage -between builds. For a proxy running on your development machine: - -```sh -PROXY_CACHE_METADATA=true proxy serve \ - -listen :8080 \ - -base-url http://host.docker.internal:8080 \ - -storage-url "file://$PWD/proxy-cache/artifacts" \ - -database-path ./proxy-cache/proxy.db -``` - -Docker Desktop provides `host.docker.internal` for reaching the host. On Linux -Docker Engine, add the host mapping when building: - -```sh -docker build \ - --add-host host.docker.internal=host-gateway \ - --build-arg PROXY_URL=http://host.docker.internal:8080 \ - -t app . -``` - -`localhost` inside a build refers to the build container. Both the build's proxy -URL and proxy's `base_url` must be reachable from that container, since some -registry responses contain download URLs derived from `base_url`. The proxy -must listen on an interface reachable from the builder, with access restricted -to the machines that should use it. - -For CI or a remote builder, use a persistent service reachable from the builder, -such as `https://proxy.example.com`, and set proxy's `base_url` to that address. -Use the same service across jobs and pass its URL with `--build-arg PROXY_URL=...`. -Creating an empty proxy for every job loses the shared cache unless its database -and storage persist. If proxy runs in a container, mount persistent storage for -both. See [configuration](configuration.md) for storage and deployment options. - -Metadata caching is optional, but avoids repeated metadata fetches within -`metadata_ttl`. Artifact caching is enabled by default. Proxy's configured -cooldown, denylist and scanning policies also apply to Docker builds. - -## npm - -In your Node build stage, copy the manifests before application source so an -unchanged dependency set can reuse Docker's installation layer: - -```dockerfile -ARG PROXY_URL -COPY package.json package-lock.json ./ -RUN npm ci --no-audit --no-fund --registry="${PROXY_URL:?PROXY_URL build arg required}/npm/" -COPY . . -``` - -Pass `PROXY_URL` without a trailing slash. With Docker Desktop: - -```sh -docker build \ - --build-arg PROXY_URL=http://host.docker.internal:8080 \ - -t app . -``` - -Keep `node_modules` out of the build context so `COPY . .` does not overwrite -the installed dependencies. Lockfiles with custom registry or download URLs -may still direct requests to those hosts; check proxy's access log when using -private registries or non-registry dependencies. `--no-audit` skips the audit -request, which proxy would otherwise forward upstream. - -## Ruby / Bundler - -Keep `https://rubygems.org` as the source in your Gemfile and configure its mirror -inside the Ruby build stage. Install any system packages needed to compile -native extensions before this step: - -```dockerfile -ARG PROXY_URL -COPY Gemfile Gemfile.lock ./ -RUN bundle config set --local mirror.https://rubygems.org "${PROXY_URL:?PROXY_URL build arg required}/gem" \ - && bundle install \ - && bundle config unset --local mirror.https://rubygems.org -COPY . . -``` - -Use the same build command as the npm example. The mirror setting is removed -after installation so it is not retained in the image, but the URL still -appears in `docker history` because Docker records the build args used by `RUN` -lines. Keep credentials out of `PROXY_URL`. Exclude host Bundler -configuration and installed gems from the build context. Copy any local path -dependencies required by the Gemfile before `bundle install`. - -The RubyGems mirror covers gems fetched from that source. Git dependencies and -other gem sources need their own configuration. Without -`mirror.https://rubygems.org.fallback_timeout`, `bundle install` fails if proxy -is unreachable. Setting it lets Bundler fall back to rubygems.org, so this setup -does not enforce network isolation either way. - -## Layer caches and package caches - -Keep existing Docker caching optimizations. If Docker reuses the installation -layer, the package manager does not run and proxy receives no requests. A -BuildKit cache mount can also preserve downloaded packages when that layer must -be rebuilt. For example, the npm step can use: - -```dockerfile -RUN --mount=type=cache,target=/root/.npm \ - npm ci --no-audit --no-fund --registry="${PROXY_URL:?PROXY_URL build arg required}/npm/" -``` - -Cache mounts are stored by the builder and are excluded from ordinary -`--cache-to` exports. Proxy can supply packages when a fresh builder has neither -the installation layer nor the package-manager cache. See Docker's -[cache guidance](https://docs.docker.com/build/cache/optimize/) and -[cache-mount reference](https://docs.docker.com/reference/dockerfile/#run---mounttypecache). - -To check the configuration, enable proxy's JSONL log with `-access-log` and -rebuild with `docker build --no-cache`. Look for requests to `/npm/` or `/gem/`, -then repeat and check `proxy_cache_hits_total` on `/metrics`. Cache mounts still -retain their contents with `--no-cache`; omit them when testing proxy cache -hits. Upstream metadata revalidation can still occur. - -## Base-image pulls - -Package-manager configuration affects commands such as `npm ci` and -`bundle install`. To cache images referenced by `FROM`, configure a registry -mirror in the builder as well. For a proxy served at -`https://proxy.example.com`, put this in `buildkitd.toml`: - -```toml -[registry."docker.io"] - mirrors = ["proxy.example.com"] -``` - -Create a Buildx builder with that configuration and select it explicitly: - -```sh -docker buildx create --name proxy-builder \ - --driver docker-container \ - --buildkitd-config ./buildkitd.toml - -docker buildx build --builder proxy-builder \ - --build-arg PROXY_URL=https://proxy.example.com \ - --load -t app . -``` - -For local Docker Desktop testing over HTTP, the configuration is: - -```toml -[registry."docker.io"] - mirrors = ["host.docker.internal:8080"] - -[registry."host.docker.internal:8080"] - http = true -``` - -The registry mirror address must be reachable from the BuildKit daemon itself. -The build's `--add-host` flag only adds a mapping inside build containers. A -separate Buildx builder has its own registry configuration; setting Docker -Engine's `registry-mirrors` does not configure that builder. See Docker's -[BuildKit configuration guide](https://docs.docker.com/build/buildkit/configure/). - -Proxy's OCI endpoint is pull-only. BuildKit registry-cache exports with -`--cache-to type=registry` need a separate writable registry. Other cache export -backends can be used alongside proxy. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index fe8723d..c4b21f3 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -15,61 +15,6 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { - "/_internal/scan-fetch": { - "get": { - "description": "Streams the exact bytes staged in storage for a pre-cache security scan.\nRequires a short-lived HMAC-signed token minted by the proxy itself and\ndelivered via the fetch_url field of the scan notify request (see the\nArtifact Scanning section of docs/configuration.md). Not part of the\npublic API; restrict access to the scanner network at the ingress layer.", - "produces": [ - "application/octet-stream" - ], - "tags": [ - "scanning" - ], - "summary": "Fetch a staged artifact for scanning", - "parameters": [ - { - "type": "string", - "description": "Storage path of the staged artifact", - "name": "path", - "in": "query", - "required": true - }, - { - "type": "integer", - "description": "Token expiry, Unix seconds", - "name": "exp", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "HMAC-SHA256 signature over the string path|exp", - "name": "sig", - "in": "query", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "file" - } - }, - "403": { - "description": "invalid, expired, or tampered token", - "schema": { - "type": "string" - } - }, - "404": { - "description": "object not found in storage, or scanning is not configured", - "schema": { - "type": "string" - } - } - } - } - }, "/api/bulk": { "post": { "consumes": [ @@ -115,253 +60,6 @@ const docTemplate = `{ } } }, - "/api/cache/{ecosystem}/{name}/{version}": { - "delete": { - "description": "Removes the cached files of name@version, or only the one named by filename, so the next request fetches them from the upstream again. Pushed archives are not touched; delete those through /api/hosted.", - "produces": [ - "application/json" - ], - "tags": [ - "cache" - ], - "summary": "Delete cached artifacts", - "parameters": [ - { - "type": "string", - "description": "Ecosystem", - "name": "ecosystem", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Package name as shown in the UI, e.g. com.google.guava:guava for Maven", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Version", - "name": "version", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Delete only this file", - "name": "filename", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/server.CacheDeleteResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, - "/api/hosted/{ecosystem}/{name}": { - "get": { - "produces": [ - "application/json" - ], - "tags": [ - "hosted" - ], - "summary": "List pushed package archives", - "parameters": [ - { - "type": "string", - "description": "Ecosystem", - "name": "ecosystem", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Package name", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/server.HostedListResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, - "/api/hosted/{ecosystem}/{name}/{version}": { - "put": { - "security": [ - { - "BearerAuth": [] - } - ], - "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted. Generic files are pushed to /api/hosted/generic/{name}/{version}/{file} and served at /generic/hosted/{name}/{version}/{file}. A file that was already pushed is refused with 409; delete it first to push it again.", - "consumes": [ - "application/octet-stream" - ], - "produces": [ - "application/json" - ], - "tags": [ - "hosted" - ], - "summary": "Push a package archive", - "parameters": [ - { - "type": "string", - "description": "Ecosystem (npm, composer or generic)", - "name": "ecosystem", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Package name, including an npm scope or the Composer vendor", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Version", - "name": "version", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Expected sha256 of the body, hex-encoded; a mismatch is refused", - "name": "sha256", - "in": "query" - } - ], - "responses": { - "201": { - "description": "Created", - "schema": { - "$ref": "#/definitions/server.HostedPushResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "413": { - "description": "Request Entity Too Large", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - }, - "delete": { - "security": [ - { - "BearerAuth": [] - } - ], - "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again. For generic files, /api/hosted/generic/{name}/{version}/{file} removes only that file.", - "tags": [ - "hosted" - ], - "summary": "Delete a pushed version", - "parameters": [ - { - "type": "string", - "description": "Ecosystem", - "name": "ecosystem", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Package name", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Version", - "name": "version", - "in": "path", - "required": true - } - ], - "responses": { - "204": { - "description": "No Content" - }, - "401": { - "description": "Unauthorized", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, "/api/outdated": { "post": { "consumes": [ @@ -460,194 +158,6 @@ const docTemplate = `{ } } }, - "/api/projects": { - "get": { - "description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.", - "produces": [ - "application/json" - ], - "tags": [ - "projects" - ], - "summary": "List projects", - "parameters": [ - { - "type": "string", - "description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20", - "name": "purl", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/server.ProjectListResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, - "/api/projects/{project}": { - "get": { - "description": "Returns the project with the package versions recorded for it.", - "produces": [ - "application/json" - ], - "tags": [ - "projects" - ], - "summary": "Get a project", - "parameters": [ - { - "type": "string", - "description": "Project path, e.g. group/sub/repo", - "name": "project", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/server.ProjectResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - }, - "delete": { - "security": [ - { - "BearerAuth": [] - } - ], - "description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.", - "tags": [ - "projects" - ], - "summary": "Delete a project", - "parameters": [ - { - "type": "string", - "description": "Project path, e.g. group/sub/repo", - "name": "project", - "in": "path", - "required": true - } - ], - "responses": { - "204": { - "description": "No Content" - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, - "/api/projects/{project}/sbom": { - "put": { - "security": [ - { - "BearerAuth": [] - } - ], - "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.", - "consumes": [ - "application/json", - "application/xml", - "text/plain" - ], - "produces": [ - "application/json" - ], - "tags": [ - "projects" - ], - "summary": "Upload a project's SBOM", - "parameters": [ - { - "type": "string", - "description": "Project path, e.g. group/sub/repo", - "name": "project", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Commit the SBOM was generated from", - "name": "ref", - "in": "query" - }, - { - "type": "string", - "description": "Branch the SBOM was generated from", - "name": "branch", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/server.ProjectUploadResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "413": { - "description": "Request Entity Too Large", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, "/api/search": { "get": { "produces": [ @@ -932,38 +442,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/server.ErrorResponse" } - }, - "503": { - "description": "Service Unavailable", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } } } } } }, "definitions": { - "handler.HostedArtifact": { - "type": "object", - "properties": { - "filename": { - "type": "string" - }, - "pushed_at": { - "type": "string" - }, - "sha256": { - "type": "string" - }, - "size": { - "type": "integer" - }, - "version": { - "type": "string" - } - } - }, "server.BrowseFileInfo": { "type": "object", "properties": { @@ -1020,49 +504,6 @@ const docTemplate = `{ } } }, - "server.CacheDeleteResponse": { - "type": "object", - "properties": { - "deleted": { - "type": "array", - "items": { - "type": "string" - } - }, - "purl": { - "type": "string" - } - } - }, - "server.EcosystemStatsEntry": { - "type": "object", - "properties": { - "cache_size_bytes": { - "type": "integer" - }, - "cached_artifacts": { - "type": "integer" - }, - "downloaded": { - "type": "string" - }, - "downloaded_bytes": { - "type": "integer" - }, - "downloads": { - "type": "integer" - }, - "ecosystem": { - "type": "string" - }, - "packages": { - "type": "integer" - }, - "versions": { - "type": "integer" - } - } - }, "server.ErrorResponse": { "type": "object", "properties": { @@ -1097,65 +538,8 @@ const docTemplate = `{ "$ref": "#/definitions/server.HealthCheck" } }, - "circuit_breakers": { - "description": "CircuitBreakers reports the state (\"open\" or \"closed\") of each upstream\nregistry's artifact-fetch circuit breaker, keyed by the host fetched from\nor, where the fetch URL has none to read, by an opaque placeholder\nstanding in for it. It is omitted when no breaker has been created yet.\nAn open breaker fails every artifact fetch it covers without contacting\nthe upstream, but says nothing about this proxy's own health, so it does\nnot change Status.", - "type": "object", - "additionalProperties": { - "type": "string" - } - }, "status": { "type": "string" - }, - "upstream_holds": { - "description": "UpstreamHolds lists upstream hosts that rate limited the proxy and\nthe RFC 3339 time until which it sends them nothing. A held host\nfails cache misses without contacting it and does not change Status.", - "type": "object", - "additionalProperties": { - "type": "string" - } - } - } - }, - "server.HostedListResponse": { - "type": "object", - "properties": { - "artifacts": { - "type": "array", - "items": { - "$ref": "#/definitions/handler.HostedArtifact" - } - }, - "ecosystem": { - "type": "string" - }, - "name": { - "type": "string" - } - } - }, - "server.HostedPushResponse": { - "type": "object", - "properties": { - "ecosystem": { - "type": "string" - }, - "filename": { - "type": "string" - }, - "name": { - "type": "string" - }, - "pushed_at": { - "type": "string" - }, - "sha256": { - "type": "string" - }, - "size": { - "type": "integer" - }, - "version": { - "type": "string" } } }, @@ -1250,10 +634,6 @@ const docTemplate = `{ "server.PackageResponse": { "type": "object", "properties": { - "cooldown": { - "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").", - "type": "string" - }, "description": { "type": "string" }, @@ -1312,106 +692,6 @@ const docTemplate = `{ } } }, - "server.ProjectComponent": { - "type": "object", - "properties": { - "ecosystem": { - "type": "string" - }, - "name": { - "type": "string" - }, - "purl": { - "type": "string" - }, - "version": { - "type": "string" - } - } - }, - "server.ProjectListResponse": { - "type": "object", - "properties": { - "projects": { - "type": "array", - "items": { - "$ref": "#/definitions/server.ProjectSummary" - } - } - } - }, - "server.ProjectResponse": { - "type": "object", - "properties": { - "branch": { - "type": "string" - }, - "component_count": { - "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", - "type": "integer" - }, - "components": { - "type": "array", - "items": { - "$ref": "#/definitions/server.ProjectComponent" - } - }, - "name": { - "type": "string" - }, - "ref": { - "type": "string" - }, - "uploaded_at": { - "type": "string" - }, - "version": { - "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", - "type": "string" - } - } - }, - "server.ProjectSummary": { - "type": "object", - "properties": { - "branch": { - "type": "string" - }, - "component_count": { - "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", - "type": "integer" - }, - "name": { - "type": "string" - }, - "ref": { - "type": "string" - }, - "uploaded_at": { - "type": "string" - }, - "version": { - "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", - "type": "string" - } - } - }, - "server.ProjectUploadResponse": { - "type": "object", - "properties": { - "components": { - "description": "Components is the number of package versions recorded.", - "type": "integer" - }, - "project": { - "type": "string" - }, - "skipped": { - "description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.", - "type": "integer" - } - } - }, "server.SearchPackageResult": { "type": "object", "properties": { @@ -1461,25 +741,11 @@ const docTemplate = `{ "cached_artifacts": { "type": "integer" }, - "downloaded": { + "database_path": { "type": "string" }, - "downloaded_bytes": { - "description": "DownloadedBytes is the accumulated download volume across every\necosystem: cache hits multiplied by the artifact size they served.", - "type": "integer" - }, - "downloads": { - "type": "integer" - }, - "ecosystems": { - "type": "array", - "items": { - "$ref": "#/definitions/server.EcosystemStatsEntry" - } - }, - "stats_unavailable": { - "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", - "type": "boolean" + "storage_url": { + "type": "string" }, "total_size": { "type": "string" @@ -1489,14 +755,6 @@ const docTemplate = `{ } } } - }, - "securityDefinitions": { - "BearerAuth": { - "description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects", - "type": "apiKey", - "name": "Authorization", - "in": "header" - } } }` diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 496aecd..898f580 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -8,61 +8,6 @@ }, "basePath": "/", "paths": { - "/_internal/scan-fetch": { - "get": { - "description": "Streams the exact bytes staged in storage for a pre-cache security scan.\nRequires a short-lived HMAC-signed token minted by the proxy itself and\ndelivered via the fetch_url field of the scan notify request (see the\nArtifact Scanning section of docs/configuration.md). Not part of the\npublic API; restrict access to the scanner network at the ingress layer.", - "produces": [ - "application/octet-stream" - ], - "tags": [ - "scanning" - ], - "summary": "Fetch a staged artifact for scanning", - "parameters": [ - { - "type": "string", - "description": "Storage path of the staged artifact", - "name": "path", - "in": "query", - "required": true - }, - { - "type": "integer", - "description": "Token expiry, Unix seconds", - "name": "exp", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "HMAC-SHA256 signature over the string path|exp", - "name": "sig", - "in": "query", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "file" - } - }, - "403": { - "description": "invalid, expired, or tampered token", - "schema": { - "type": "string" - } - }, - "404": { - "description": "object not found in storage, or scanning is not configured", - "schema": { - "type": "string" - } - } - } - } - }, "/api/bulk": { "post": { "consumes": [ @@ -108,253 +53,6 @@ } } }, - "/api/cache/{ecosystem}/{name}/{version}": { - "delete": { - "description": "Removes the cached files of name@version, or only the one named by filename, so the next request fetches them from the upstream again. Pushed archives are not touched; delete those through /api/hosted.", - "produces": [ - "application/json" - ], - "tags": [ - "cache" - ], - "summary": "Delete cached artifacts", - "parameters": [ - { - "type": "string", - "description": "Ecosystem", - "name": "ecosystem", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Package name as shown in the UI, e.g. com.google.guava:guava for Maven", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Version", - "name": "version", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Delete only this file", - "name": "filename", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/server.CacheDeleteResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, - "/api/hosted/{ecosystem}/{name}": { - "get": { - "produces": [ - "application/json" - ], - "tags": [ - "hosted" - ], - "summary": "List pushed package archives", - "parameters": [ - { - "type": "string", - "description": "Ecosystem", - "name": "ecosystem", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Package name", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/server.HostedListResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, - "/api/hosted/{ecosystem}/{name}/{version}": { - "put": { - "security": [ - { - "BearerAuth": [] - } - ], - "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted. Generic files are pushed to /api/hosted/generic/{name}/{version}/{file} and served at /generic/hosted/{name}/{version}/{file}. A file that was already pushed is refused with 409; delete it first to push it again.", - "consumes": [ - "application/octet-stream" - ], - "produces": [ - "application/json" - ], - "tags": [ - "hosted" - ], - "summary": "Push a package archive", - "parameters": [ - { - "type": "string", - "description": "Ecosystem (npm, composer or generic)", - "name": "ecosystem", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Package name, including an npm scope or the Composer vendor", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Version", - "name": "version", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Expected sha256 of the body, hex-encoded; a mismatch is refused", - "name": "sha256", - "in": "query" - } - ], - "responses": { - "201": { - "description": "Created", - "schema": { - "$ref": "#/definitions/server.HostedPushResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "413": { - "description": "Request Entity Too Large", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - }, - "delete": { - "security": [ - { - "BearerAuth": [] - } - ], - "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again. For generic files, /api/hosted/generic/{name}/{version}/{file} removes only that file.", - "tags": [ - "hosted" - ], - "summary": "Delete a pushed version", - "parameters": [ - { - "type": "string", - "description": "Ecosystem", - "name": "ecosystem", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Package name", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Version", - "name": "version", - "in": "path", - "required": true - } - ], - "responses": { - "204": { - "description": "No Content" - }, - "401": { - "description": "Unauthorized", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, "/api/outdated": { "post": { "consumes": [ @@ -453,194 +151,6 @@ } } }, - "/api/projects": { - "get": { - "description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.", - "produces": [ - "application/json" - ], - "tags": [ - "projects" - ], - "summary": "List projects", - "parameters": [ - { - "type": "string", - "description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20", - "name": "purl", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/server.ProjectListResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, - "/api/projects/{project}": { - "get": { - "description": "Returns the project with the package versions recorded for it.", - "produces": [ - "application/json" - ], - "tags": [ - "projects" - ], - "summary": "Get a project", - "parameters": [ - { - "type": "string", - "description": "Project path, e.g. group/sub/repo", - "name": "project", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/server.ProjectResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - }, - "delete": { - "security": [ - { - "BearerAuth": [] - } - ], - "description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.", - "tags": [ - "projects" - ], - "summary": "Delete a project", - "parameters": [ - { - "type": "string", - "description": "Project path, e.g. group/sub/repo", - "name": "project", - "in": "path", - "required": true - } - ], - "responses": { - "204": { - "description": "No Content" - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, - "/api/projects/{project}/sbom": { - "put": { - "security": [ - { - "BearerAuth": [] - } - ], - "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.", - "consumes": [ - "application/json", - "application/xml", - "text/plain" - ], - "produces": [ - "application/json" - ], - "tags": [ - "projects" - ], - "summary": "Upload a project's SBOM", - "parameters": [ - { - "type": "string", - "description": "Project path, e.g. group/sub/repo", - "name": "project", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Commit the SBOM was generated from", - "name": "ref", - "in": "query" - }, - { - "type": "string", - "description": "Branch the SBOM was generated from", - "name": "branch", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/server.ProjectUploadResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - }, - "413": { - "description": "Request Entity Too Large", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } - } - } - } - }, "/api/search": { "get": { "produces": [ @@ -925,38 +435,12 @@ "schema": { "$ref": "#/definitions/server.ErrorResponse" } - }, - "503": { - "description": "Service Unavailable", - "schema": { - "$ref": "#/definitions/server.ErrorResponse" - } } } } } }, "definitions": { - "handler.HostedArtifact": { - "type": "object", - "properties": { - "filename": { - "type": "string" - }, - "pushed_at": { - "type": "string" - }, - "sha256": { - "type": "string" - }, - "size": { - "type": "integer" - }, - "version": { - "type": "string" - } - } - }, "server.BrowseFileInfo": { "type": "object", "properties": { @@ -1013,49 +497,6 @@ } } }, - "server.CacheDeleteResponse": { - "type": "object", - "properties": { - "deleted": { - "type": "array", - "items": { - "type": "string" - } - }, - "purl": { - "type": "string" - } - } - }, - "server.EcosystemStatsEntry": { - "type": "object", - "properties": { - "cache_size_bytes": { - "type": "integer" - }, - "cached_artifacts": { - "type": "integer" - }, - "downloaded": { - "type": "string" - }, - "downloaded_bytes": { - "type": "integer" - }, - "downloads": { - "type": "integer" - }, - "ecosystem": { - "type": "string" - }, - "packages": { - "type": "integer" - }, - "versions": { - "type": "integer" - } - } - }, "server.ErrorResponse": { "type": "object", "properties": { @@ -1090,65 +531,8 @@ "$ref": "#/definitions/server.HealthCheck" } }, - "circuit_breakers": { - "description": "CircuitBreakers reports the state (\"open\" or \"closed\") of each upstream\nregistry's artifact-fetch circuit breaker, keyed by the host fetched from\nor, where the fetch URL has none to read, by an opaque placeholder\nstanding in for it. It is omitted when no breaker has been created yet.\nAn open breaker fails every artifact fetch it covers without contacting\nthe upstream, but says nothing about this proxy's own health, so it does\nnot change Status.", - "type": "object", - "additionalProperties": { - "type": "string" - } - }, "status": { "type": "string" - }, - "upstream_holds": { - "description": "UpstreamHolds lists upstream hosts that rate limited the proxy and\nthe RFC 3339 time until which it sends them nothing. A held host\nfails cache misses without contacting it and does not change Status.", - "type": "object", - "additionalProperties": { - "type": "string" - } - } - } - }, - "server.HostedListResponse": { - "type": "object", - "properties": { - "artifacts": { - "type": "array", - "items": { - "$ref": "#/definitions/handler.HostedArtifact" - } - }, - "ecosystem": { - "type": "string" - }, - "name": { - "type": "string" - } - } - }, - "server.HostedPushResponse": { - "type": "object", - "properties": { - "ecosystem": { - "type": "string" - }, - "filename": { - "type": "string" - }, - "name": { - "type": "string" - }, - "pushed_at": { - "type": "string" - }, - "sha256": { - "type": "string" - }, - "size": { - "type": "integer" - }, - "version": { - "type": "string" } } }, @@ -1243,10 +627,6 @@ "server.PackageResponse": { "type": "object", "properties": { - "cooldown": { - "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").", - "type": "string" - }, "description": { "type": "string" }, @@ -1305,106 +685,6 @@ } } }, - "server.ProjectComponent": { - "type": "object", - "properties": { - "ecosystem": { - "type": "string" - }, - "name": { - "type": "string" - }, - "purl": { - "type": "string" - }, - "version": { - "type": "string" - } - } - }, - "server.ProjectListResponse": { - "type": "object", - "properties": { - "projects": { - "type": "array", - "items": { - "$ref": "#/definitions/server.ProjectSummary" - } - } - } - }, - "server.ProjectResponse": { - "type": "object", - "properties": { - "branch": { - "type": "string" - }, - "component_count": { - "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", - "type": "integer" - }, - "components": { - "type": "array", - "items": { - "$ref": "#/definitions/server.ProjectComponent" - } - }, - "name": { - "type": "string" - }, - "ref": { - "type": "string" - }, - "uploaded_at": { - "type": "string" - }, - "version": { - "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", - "type": "string" - } - } - }, - "server.ProjectSummary": { - "type": "object", - "properties": { - "branch": { - "type": "string" - }, - "component_count": { - "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", - "type": "integer" - }, - "name": { - "type": "string" - }, - "ref": { - "type": "string" - }, - "uploaded_at": { - "type": "string" - }, - "version": { - "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", - "type": "string" - } - } - }, - "server.ProjectUploadResponse": { - "type": "object", - "properties": { - "components": { - "description": "Components is the number of package versions recorded.", - "type": "integer" - }, - "project": { - "type": "string" - }, - "skipped": { - "description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.", - "type": "integer" - } - } - }, "server.SearchPackageResult": { "type": "object", "properties": { @@ -1454,25 +734,11 @@ "cached_artifacts": { "type": "integer" }, - "downloaded": { + "database_path": { "type": "string" }, - "downloaded_bytes": { - "description": "DownloadedBytes is the accumulated download volume across every\necosystem: cache hits multiplied by the artifact size they served.", - "type": "integer" - }, - "downloads": { - "type": "integer" - }, - "ecosystems": { - "type": "array", - "items": { - "$ref": "#/definitions/server.EcosystemStatsEntry" - } - }, - "stats_unavailable": { - "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", - "type": "boolean" + "storage_url": { + "type": "string" }, "total_size": { "type": "string" @@ -1482,13 +748,5 @@ } } } - }, - "securityDefinitions": { - "BearerAuth": { - "description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects", - "type": "apiKey", - "name": "Authorization", - "in": "header" - } } } \ No newline at end of file diff --git a/go.mod b/go.mod index 9b77ac2..471ab53 100644 --- a/go.mod +++ b/go.mod @@ -1,131 +1,318 @@ module github.com/git-pkgs/proxy -go 1.27.3 +go 1.25.6 require ( github.com/BurntSushi/toml v1.6.0 - github.com/Masterminds/semver/v3 v3.5.0 - github.com/aws/aws-sdk-go-v2/config v1.33.12 - github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1 - github.com/git-pkgs/archives v0.10.0 - github.com/git-pkgs/artifacts v0.2.1 - github.com/git-pkgs/cooldown v0.3.1 - github.com/git-pkgs/enrichment v0.7.5 - github.com/git-pkgs/gcs v0.1.0 - github.com/git-pkgs/integrity v0.2.0 - github.com/git-pkgs/magic v0.6.0 - github.com/git-pkgs/purl v0.2.1 - github.com/git-pkgs/registries v0.13.0 - github.com/git-pkgs/sbom v0.1.9 - github.com/git-pkgs/spdx v0.3.3 - github.com/git-pkgs/vers v0.7.2 - github.com/go-chi/chi/v5 v5.5.0 + github.com/CycloneDX/cyclonedx-go v0.11.0 + github.com/git-pkgs/archives v0.3.0 + github.com/git-pkgs/cooldown v0.1.1 + github.com/git-pkgs/enrichment v0.4.1 + github.com/git-pkgs/purl v0.1.13 + github.com/git-pkgs/registries v0.6.2 + github.com/git-pkgs/spdx v0.1.4 + github.com/git-pkgs/vers v0.2.6 + github.com/git-pkgs/vulns v0.1.6 + github.com/go-chi/chi/v5 v5.3.0 github.com/jmoiron/sqlx v1.4.0 - github.com/klauspost/compress v1.20.3 github.com/lib/pq v1.12.3 - github.com/opencontainers/go-digest v1.0.0 - github.com/prometheus/client_golang v1.25.0 - github.com/prometheus/client_model v0.6.3 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 + github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 - github.com/vmihailenco/msgpack/v5 v5.4.1 - gocloud.dev v0.47.0 - golang.org/x/mod v0.42.0 - golang.org/x/net v0.62.0 - golang.org/x/sync v0.24.0 - google.golang.org/protobuf v1.36.12 + gocloud.dev v0.46.0 + golang.org/x/sync v0.21.0 + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.61.0 + modernc.org/sqlite v1.53.0 ) require ( - cloud.google.com/go/auth v0.23.2 // indirect + 4d63.com/gocheckcompilerdirectives v1.3.0 // indirect + 4d63.com/gochecknoglobals v0.2.2 // indirect + cloud.google.com/go/auth v0.18.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 // indirect + codeberg.org/chavacava/garif v0.2.0 // indirect + codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect + dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect + dev.gaijin.team/go/golib v0.6.0 // indirect + github.com/4meepo/tagalign v1.4.3 // indirect + github.com/Abirdcfly/dupword v0.1.7 // indirect + github.com/AdminBenni/iota-mixing v1.0.0 // indirect + github.com/AlwxSin/noinlineerr v1.0.5 // indirect + github.com/Antonboom/errname v1.1.1 // indirect + github.com/Antonboom/nilnil v1.1.1 // indirect + github.com/Antonboom/testifylint v1.6.4 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 // indirect + github.com/Djarvur/go-err113 v0.1.1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/MirrexOne/unqueryvet v1.5.3 // indirect + github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/alecthomas/chroma/v2 v2.23.1 // indirect + github.com/alecthomas/go-check-sumtype v0.3.1 // indirect + github.com/alexkohler/nakedret/v2 v2.0.6 // indirect + github.com/alexkohler/prealloc v1.0.2 // indirect + github.com/alfatraining/structtag v1.0.0 // indirect + github.com/alingse/asasalint v0.0.11 // indirect + github.com/alingse/nilnesserr v0.2.0 // indirect + github.com/anchore/go-struct-converter v0.1.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.47.5 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.20.12 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect - github.com/aws/smithy-go v1.28.7 // indirect + github.com/ashanbrown/forbidigo/v2 v2.3.0 // indirect + github.com/ashanbrown/makezero/v2 v2.1.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.9 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.20 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.19 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 // indirect + github.com/aws/smithy-go v1.26.0 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bkielbasa/cyclop v1.2.3 // indirect + github.com/blizzy78/varnamelen v0.8.0 // indirect + github.com/bombsimon/wsl/v4 v4.7.0 // indirect + github.com/bombsimon/wsl/v5 v5.6.0 // indirect + github.com/breml/bidichk v0.3.3 // indirect + github.com/breml/errchkjson v0.4.1 // indirect + github.com/butuzov/ireturn v0.4.0 // indirect + github.com/butuzov/mirror v1.3.0 // indirect + github.com/catenacyber/perfsprint v0.10.1 // indirect + github.com/ccojocar/zxcvbn-go v1.0.4 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/dustin/go-humanize v1.1.0 // indirect - github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect + github.com/charithe/durationcheck v0.0.11 // indirect + github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect + github.com/charmbracelet/lipgloss v1.1.0 // indirect + github.com/charmbracelet/x/ansi v0.10.1 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/ckaznocha/intrange v0.3.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect + github.com/curioswitch/go-reassign v0.3.0 // indirect + github.com/daixiang0/gci v0.13.7 // indirect + github.com/dave/dst v0.27.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/denis-tingaikin/go-header v0.5.0 // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ecosyste-ms/ecosystems-go v0.2.0 // indirect + github.com/ettle/strcase v0.2.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect - github.com/git-pkgs/pom v0.1.8 // indirect - github.com/git-pkgs/vulns v0.2.4 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/fatih/structtag v1.2.0 // indirect + github.com/firefart/nonamedreturns v1.0.6 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fzipp/gocyclo v0.6.0 // indirect + github.com/ghostiam/protogetter v0.3.20 // indirect + github.com/git-pkgs/packageurl-go v0.3.1 // indirect + github.com/git-pkgs/pom v0.1.5 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect - github.com/go-logr/logr v1.4.4 // indirect + github.com/go-critic/go-critic v0.14.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/spec v0.20.4 // indirect github.com/go-openapi/swag v0.19.15 // indirect + github.com/go-toolsmith/astcast v1.1.0 // indirect + github.com/go-toolsmith/astcopy v1.1.0 // indirect + github.com/go-toolsmith/astequal v1.2.0 // indirect + github.com/go-toolsmith/astfmt v1.1.0 // indirect + github.com/go-toolsmith/astp v1.1.0 // indirect + github.com/go-toolsmith/strparse v1.1.0 // indirect + github.com/go-toolsmith/typep v1.1.0 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/godoc-lint/godoc-lint v0.11.2 // indirect + github.com/gofrs/flock v0.13.0 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/golangci/asciicheck v0.5.0 // indirect + github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 // indirect + github.com/golangci/go-printf-func-name v0.1.1 // indirect + github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d // indirect + github.com/golangci/golangci-lint/v2 v2.10.1 // indirect + github.com/golangci/golines v0.15.0 // indirect + github.com/golangci/misspell v0.8.0 // indirect + github.com/golangci/plugin-module-register v0.1.2 // indirect + github.com/golangci/revgrep v0.8.0 // indirect + github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect + github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect github.com/google/wire v0.7.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect - github.com/googleapis/gax-go/v2 v2.24.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect + github.com/googleapis/gax-go/v2 v2.19.0 // indirect + github.com/gordonklaus/ineffassign v0.2.0 // indirect + github.com/gostaticanalysis/analysisutil v0.7.1 // indirect + github.com/gostaticanalysis/comment v1.5.0 // indirect + github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect + github.com/gostaticanalysis/nilerr v0.1.2 // indirect + github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect + github.com/hashicorp/go-version v1.8.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hexops/gotextdiff v1.0.3 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jgautheron/goconst v1.8.2 // indirect + github.com/jingyugao/rowserrcheck v1.1.1 // indirect + github.com/jjti/go-spancheck v0.6.5 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/julz/importas v0.2.0 // indirect + github.com/karamaru-alpha/copyloopvar v1.2.2 // indirect + github.com/kisielk/errcheck v1.9.0 // indirect + github.com/kkHAIKE/contextcheck v1.1.6 // indirect + github.com/kulti/thelper v0.7.1 // indirect + github.com/kunwardeep/paralleltest v1.0.15 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lasiar/canonicalheader v1.1.2 // indirect + github.com/ldez/exptostd v0.4.5 // indirect + github.com/ldez/gomoddirectives v0.8.0 // indirect + github.com/ldez/grignotin v0.10.1 // indirect + github.com/ldez/structtags v0.6.1 // indirect + github.com/ldez/tagliatelle v0.7.2 // indirect + github.com/ldez/usetesting v0.5.0 // indirect + github.com/leonklingele/grouper v1.1.2 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/macabu/inamedparam v0.2.0 // indirect + github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-isatty v0.0.24 // indirect + github.com/manuelarte/embeddedstructfieldcheck v0.4.0 // indirect + github.com/manuelarte/funcorder v0.5.0 // indirect + github.com/maratori/testableexamples v1.0.1 // indirect + github.com/maratori/testpackage v1.1.2 // indirect + github.com/matoous/godox v1.1.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mgechev/revive v1.14.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moricho/tparallel v0.3.2 // indirect + github.com/muesli/termenv v0.16.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/ncruces/go-strftime v1.1.0 // indirect - github.com/oapi-codegen/nullable v1.2.0 // indirect - github.com/oapi-codegen/runtime v1.7.0 // indirect - github.com/package-url/packageurl-go v0.1.7 // indirect - github.com/pandatix/go-cvss v0.6.4 // indirect + github.com/nakabonne/nestif v0.3.1 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/nishanths/exhaustive v0.12.0 // indirect + github.com/nishanths/predeclared v0.2.2 // indirect + github.com/nunnatsa/ginkgolinter v0.23.0 // indirect + github.com/oapi-codegen/runtime v1.4.1 // indirect + github.com/package-url/packageurl-go v0.1.6 // indirect + github.com/pandatix/go-cvss v0.6.2 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/prometheus/common v0.72.0 // indirect - github.com/prometheus/procfs v0.22.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.20.1 // indirect + github.com/quasilyte/go-ruleguard v0.4.5 // indirect + github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect + github.com/quasilyte/gogrep v0.5.0 // indirect + github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect + github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect + github.com/raeperd/recvcheck v0.2.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rogpeppe/go-internal v1.16.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect - github.com/ulikunitz/xz v0.5.17 // indirect - github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/ryancurrah/gomodguard v1.4.1 // indirect + github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect + github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/sashamelentyev/interfacebloat v1.1.0 // indirect + github.com/sashamelentyev/usestdlibvars v1.29.0 // indirect + github.com/securego/gosec/v2 v2.23.0 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + github.com/sivchari/containedctx v1.0.3 // indirect + github.com/sonatard/noctx v0.4.0 // indirect + github.com/sourcegraph/go-diff v0.7.0 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/spf13/viper v1.12.0 // indirect + github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect + github.com/stbenjam/no-sprintf-host-port v0.3.1 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/subosito/gotenv v1.4.1 // indirect + github.com/tetafro/godot v1.5.4 // indirect + github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 // indirect + github.com/timonwong/loggercheck v0.11.0 // indirect + github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect + github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect + github.com/ultraware/funlen v0.2.0 // indirect + github.com/ultraware/whitespace v0.2.0 // indirect + github.com/urfave/cli/v2 v2.3.0 // indirect + github.com/uudashr/gocognit v1.2.0 // indirect + github.com/uudashr/iface v1.4.1 // indirect + github.com/xen0n/gosmopolitan v1.3.0 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yagipy/maintidx v1.0.0 // indirect + github.com/yeya24/promlinter v0.3.0 // indirect + github.com/ykadowak/zerologlint v0.1.5 // indirect + gitlab.com/bosi/decorder v0.4.2 // indirect + go-simpler.org/musttag v0.14.0 // indirect + go-simpler.org/sloglint v0.11.1 // indirect + go.augendre.info/arangolint v0.4.0 // indirect + go.augendre.info/fatcontext v0.9.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.46.0 // indirect - go.opentelemetry.io/otel/metric v1.46.0 // indirect - go.opentelemetry.io/otel/sdk v1.46.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect - go.opentelemetry.io/otel/trace v1.46.0 // indirect - go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/crypto v0.58.0 // indirect - golang.org/x/oauth2 v0.37.0 // indirect - golang.org/x/sys v0.49.0 // indirect - golang.org/x/text v0.43.0 // indirect - golang.org/x/tools v0.52.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.51.0 // indirect + golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect + golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/net v0.54.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + golang.org/x/tools v0.45.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/api v0.297.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4 // indirect - google.golang.org/grpc v1.83.2 // indirect + google.golang.org/api v0.272.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 // indirect + google.golang.org/grpc v1.81.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - modernc.org/libc v1.79.0 // indirect + honnef.co/go/tools v0.7.0 // indirect + modernc.org/libc v1.73.4 // indirect modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.12.1 // indirect + modernc.org/memory v1.11.0 // indirect + mvdan.cc/gofumpt v0.9.2 // indirect + mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) + +tool github.com/golangci/golangci-lint/v2/cmd/golangci-lint diff --git a/go.sum b/go.sum index cb9226b..8d8c96d 100644 --- a/go.sum +++ b/go.sum @@ -1,159 +1,280 @@ -cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= -cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +4d63.com/gocheckcompilerdirectives v1.3.0 h1:Ew5y5CtcAAQeTVKUVFrE7EwHMrTO6BggtEj8BZSjZ3A= +4d63.com/gocheckcompilerdirectives v1.3.0/go.mod h1:ofsJ4zx2QAuIP/NO/NAh1ig6R1Fb18/GI7RVMwz7kAY= +4d63.com/gochecknoglobals v0.2.2 h1:H1vdnwnMaZdQW/N+NrkT1SZMTBmcwHe9Vq8lJcYYTtU= +4d63.com/gochecknoglobals v0.2.2/go.mod h1:lLxwTQjL5eIesRbvnzIP3jZtG140FnTdz+AlMa+ogt0= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.23.2 h1:pxSCpfiji41hpzpPdMCftEUCezpgpqmmDdYiAjCKXxo= -cloud.google.com/go/auth v0.23.2/go.mod h1:4DhBRcqvtljQN3dJ57qtqbib5ZGCYE5f2crfiiC2EM0= +cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM= +cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.13.0 h1:ufT3FPT5rFFXu6UtLkNoxaOaV5EuA1dsSkmemCSTo6U= -cloud.google.com/go/iam v1.13.0/go.mod h1:gHXdDEiPDvqd1q1KwBDGQlgZY/BwY760zU2LhOZS5w0= -cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= -cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= -cloud.google.com/go/storage v1.67.0 h1:8xD3NvKuHE4c+b/kmSIVBdTc0BiOjplx+KecKGR9fr8= -cloud.google.com/go/storage v1.67.0/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.61.3 h1:VS//ZfBuPGDvakfD9xyPW1RGF1Vy3BWUoVZXgW1KMOg= +cloud.google.com/go/storage v1.61.3/go.mod h1:JtqK8BBB7TWv0HVGHubtUdzYYrakOQIsMLffZ2Z/HWk= +codeberg.org/chavacava/garif v0.2.0 h1:F0tVjhYbuOCnvNcU3YSpO6b3Waw6Bimy4K0mM8y6MfY= +codeberg.org/chavacava/garif v0.2.0/go.mod h1:P2BPbVbT4QcvLZrORc2T29szK3xEOlnl0GiPTJmEqBQ= +codeberg.org/polyfloyd/go-errorlint v1.9.0 h1:VkdEEmA1VBpH6ecQoMR4LdphVI3fA4RrCh2an7YmodI= +codeberg.org/polyfloyd/go-errorlint v1.9.0/go.mod h1:GPRRu2LzVijNn4YkrZYJfatQIdS+TrcK8rL5Xs24qw8= +dev.gaijin.team/go/exhaustruct/v4 v4.0.0 h1:873r7aNneqoBB3IaFIzhvt2RFYTuHgmMjoKfwODoI1Y= +dev.gaijin.team/go/exhaustruct/v4 v4.0.0/go.mod h1:aZ/k2o4Y05aMJtiux15x8iXaumE88YdiB0Ai4fXOzPI= +dev.gaijin.team/go/golib v0.6.0 h1:v6nnznFTs4bppib/NyU1PQxobwDHwCXXl15P7DV5Zgo= +dev.gaijin.team/go/golib v0.6.0/go.mod h1:uY1mShx8Z/aNHWDyAkZTkX+uCi5PdX7KsG1eDQa2AVE= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/4meepo/tagalign v1.4.3 h1:Bnu7jGWwbfpAie2vyl63Zup5KuRv21olsPIha53BJr8= +github.com/4meepo/tagalign v1.4.3/go.mod h1:00WwRjiuSbrRJnSVeGWPLp2epS5Q/l4UEy0apLLS37c= +github.com/Abirdcfly/dupword v0.1.7 h1:2j8sInznrje4I0CMisSL6ipEBkeJUJAmK1/lfoNGWrQ= +github.com/Abirdcfly/dupword v0.1.7/go.mod h1:K0DkBeOebJ4VyOICFdppB23Q0YMOgVafM0zYW0n9lF4= +github.com/AdminBenni/iota-mixing v1.0.0 h1:Os6lpjG2dp/AE5fYBPAA1zfa2qMdCAWwPMCgpwKq7wo= +github.com/AdminBenni/iota-mixing v1.0.0/go.mod h1:i4+tpAaB+qMVIV9OK3m4/DAynOd5bQFaOu+2AhtBCNY= +github.com/AlwxSin/noinlineerr v1.0.5 h1:RUjt63wk1AYWTXtVXbSqemlbVTb23JOSRiNsshj7TbY= +github.com/AlwxSin/noinlineerr v1.0.5/go.mod h1:+QgkkoYrMH7RHvcdxdlI7vYYEdgeoFOVjU9sUhw/rQc= +github.com/Antonboom/errname v1.1.1 h1:bllB7mlIbTVzO9jmSWVWLjxTEbGBVQ1Ff/ClQgtPw9Q= +github.com/Antonboom/errname v1.1.1/go.mod h1:gjhe24xoxXp0ScLtHzjiXp0Exi1RFLKJb0bVBtWKCWQ= +github.com/Antonboom/nilnil v1.1.1 h1:9Mdr6BYd8WHCDngQnNVV0b554xyisFioEKi30sksufQ= +github.com/Antonboom/nilnil v1.1.1/go.mod h1:yCyAmSw3doopbOWhJlVci+HuyNRuHJKIv6V2oYQa8II= +github.com/Antonboom/testifylint v1.6.4 h1:gs9fUEy+egzxkEbq9P4cpcMB6/G0DYdMeiFS87UiqmQ= +github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 h1:MDT4FxAPve5FnYn6vOL1r7RCRDG+l9cI7a5LlCuHsqA= -github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 h1:4iB+IesclUXdP0ICgAabvq2FYLXrJWKx1fJQ+GxSo3Y= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.37.0 h1:edOuLWehuDlQ68roBF900Y09jvBoJP5W5d/wusz9nWY= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.37.0/go.mod h1:iIyrdhveU5Ow84ipRlxtjCOQnIM0DiqSMxuxm50xwZY= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0 h1:BeC1Bub7Ttk33TF8v3ZeQ2L8acvz4ZLqtQuLyJAZquc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0/go.mod h1:UgG/Xn9+NXb1/ueSczIFxyuya3hs3srFDiZG2JrpJwY= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0 h1:KhpL26/GviYITHMCEGGqT/l50siFaBCBBKrjv6Z0G/k= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y= +github.com/CycloneDX/cyclonedx-go v0.11.0 h1:GokP8FiRC+foiuwWhSSLpSD5H4hSWtGnR3wo7apkBFI= +github.com/CycloneDX/cyclonedx-go v0.11.0/go.mod h1:vUvbCXQsEm48OI6oOlanxstwNByXjCZ2wuleUlwGEO8= +github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g= +github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= -github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/MirrexOne/unqueryvet v1.5.3 h1:LpT3rsH+IY3cQddWF9bg4C7jsbASdGnrOSofY8IPEiw= +github.com/MirrexOne/unqueryvet v1.5.3/go.mod h1:fs9Zq6eh1LRIhsDIsxf9PONVUjYdFHdtkHIgZdJnyPU= +github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= +github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/chroma/v2 v2.23.1 h1:nv2AVZdTyClGbVQkIzlDm/rnhk1E9bU9nXwmZ/Vk/iY= +github.com/alecthomas/chroma/v2 v2.23.1/go.mod h1:NqVhfBR0lte5Ouh3DcthuUCTUpDC9cxBOfyMbMQPs3o= +github.com/alecthomas/go-check-sumtype v0.3.1 h1:u9aUvbGINJxLVXiFvHUlPEaD7VDULsrxJb4Aq31NLkU= +github.com/alecthomas/go-check-sumtype v0.3.1/go.mod h1:A8TSiN3UPRw3laIgWEUOHHLPa6/r9MtoigdlP5h3K/E= +github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= +github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/alexkohler/nakedret/v2 v2.0.6 h1:ME3Qef1/KIKr3kWX3nti3hhgNxw6aqN5pZmQiFSsuzQ= +github.com/alexkohler/nakedret/v2 v2.0.6/go.mod h1:l3RKju/IzOMQHmsEvXwkqMDzHHvurNQfAgE1eVmT40Q= +github.com/alexkohler/prealloc v1.0.2 h1:MPo8cIkGkZytq7WNH9UHv3DIX1mPz1RatPXnZb0zHWQ= +github.com/alexkohler/prealloc v1.0.2/go.mod h1:fT39Jge3bQrfA7nPMDngUfvUbQGQeJyGQnR+913SCig= +github.com/alfatraining/structtag v1.0.0 h1:2qmcUqNcCoyVJ0up879K614L9PazjBSFruTB0GOFjCc= +github.com/alfatraining/structtag v1.0.0/go.mod h1:p3Xi5SwzTi+Ryj64DqjLWz7XurHxbGsq6y3ubePJPus= +github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw= +github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= +github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEWd/w= +github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= +github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9dedlWa0OhD30= +github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= -github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= -github.com/aws/aws-sdk-go-v2/config v1.33.12 h1:IV1YARaXahwCqUIyP8IlWsP85Lil+5ozTozDQXPumRk= -github.com/aws/aws-sdk-go-v2/config v1.33.12/go.mod h1:h51R06bzMVwNBRZOX5ylHhNDnkB/azZeq/LvxM6q2pQ= -github.com/aws/aws-sdk-go-v2/credentials v1.20.12 h1:6P7Q7yH/GIwHg6MLKv3ySsBqaCoE1z+C0OTzYMenhEY= -github.com/aws/aws-sdk-go-v2/credentials v1.20.12/go.mod h1:Kw/SaoW/wc7IqJErqXJIzKo8b7Aelxfbuc6XlbrRnsc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 h1:vsFkmFrfzODkxVbyuPLscv5j5lC8KbG48LZwV+p56Y0= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3/go.mod h1:pjQYVtvVBEtlsf2p6nXMsjPuDaWs9GM44FMdHbmnmYU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= -github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1 h1:EJlz0wSlZDKaCN2FHwg9RBR+C+gsuVdYblJnEnrq3M4= -github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1/go.mod h1:6rVJFO9ZO+4j/QL5/FbtAsbb8xHC85j79wXA9ndOCTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 h1:CO485zrLOZBeq8mrdsudc8cjyhvC4LY4JFt8ZfHvXZ0= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2/go.mod h1:cBAjVUKBr+5wnx1K3XWS8u5AZsqpA2S/0z7YtI6BGFc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 h1:Enx5JcIy/o90kio1LFZ/HguxIsos3clbGuSpGdyshX8= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2/go.mod h1:Ldmll8/2VqaVerWWDf8IGG+6pAvZY04gs3tevq3nIFs= -github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 h1:4PokON/ISKuD6Zl062n4WfQNnEFBMHvfeJy73Jp7K+g= -github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0/go.mod h1:6ZFrmv/qIAYFmG+2WhA7mFU8+edldRlISwT+tNnyLuA= -github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 h1:3vu/eUqh0rjh7Zy4N4YKcpZX4Aud+zinB3VsLhhlQFw= -github.com/aws/aws-sdk-go-v2/service/signin v1.12.0/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y= -github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 h1:kIEpD5dUry7VET3Ybl7zj7PPHRB4lb52G9iTz+a1Tqk= -github.com/aws/aws-sdk-go-v2/service/sso v1.39.0/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 h1:WfFev+HgTodggcNon5HieiLxxwWwb8Br8R2+nDkixvk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw= -github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= -github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/ashanbrown/forbidigo/v2 v2.3.0 h1:OZZDOchCgsX5gvToVtEBoV2UWbFfI6RKQTir2UZzSxo= +github.com/ashanbrown/forbidigo/v2 v2.3.0/go.mod h1:5p6VmsG5/1xx3E785W9fouMxIOkvY2rRV9nMdWadd6c= +github.com/ashanbrown/makezero/v2 v2.1.0 h1:snuKYMbqosNokUKm+R6/+vOPs8yVAi46La7Ck6QYSaE= +github.com/ashanbrown/makezero/v2 v2.1.0/go.mod h1:aEGT/9q3S8DHeE57C88z2a6xydvgx8J5hgXIGWgo0MY= +github.com/aws/aws-sdk-go-v2 v1.41.9 h1:/rYeyO2+HrMztAmxAq9++XJtFMqSIpSsNA0yDGALYq4= +github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94= +github.com/aws/aws-sdk-go-v2/config v1.32.20 h1:8VMDnWc/kEzxsI/1ngGM9mG81a8IGmIHD8KLcYGwagc= +github.com/aws/aws-sdk-go-v2/config v1.32.20/go.mod h1:PuwEpciweIXGULWeOeSTXtSbH4CW9mWdWrhdCKQI1sM= +github.com/aws/aws-sdk-go-v2/credentials v1.19.19 h1:yuFzSV1U0aRNYCQGVaTY2zW2M/L93pYHnXnrJUphYhU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.19/go.mod h1:7y63L1kGzeoDlJaQ3Z578KrnmfBut96JjvJUzGwR+YE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 h1:0w6dCiO8iez+YKwRhRBlL1CH/E3GTfdkuzrwj1by8vo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25/go.mod h1:9FDWUothyr5RCRAHc45XOiVCzUR8n/IhCYX+uVqw6vk= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 h1:w5OoDiMN6x53ROmiIImGzmVcxXv2q1GXY+aKV4WAJYM= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3/go.mod h1:dAhgYp776bX3LuWvnSCFwQEjNs6fuFg7YXIy5PXcP3Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 h1:Uii3frf9ztec/ABM2/FSH9/z7PLzxfpG8h4RpkUFflQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25/go.mod h1:G6kntsA2GorAxDPbap6xgB2F+amSLUF8GJTi7PUoX44= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 h1:r1+/l6m+WaUJF9HISEsNOLHSNj5EXYQxK8VX6Cz9NlA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25/go.mod h1:cKf+D+NMDK1LndD7BowHbBZPgR9V0/5HubH0PFWvA+c= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 h1:A1PmWU2zfkIm9EyFlJncFXL4W4phML+h8KjltUsCvNQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26/go.mod h1:dY4MRzXEizrD4hqtpKvWVGPX7QleSGGVY+EBolo1RmM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 h1:d5/908OJ4bXg8lyjeMPvXetEKqoDoLi5Owy1zNue3yg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10/go.mod h1:a57l7Hwh+FWI+we50g5NPJHYUKeJKfXbc4w8SyXu8Ig= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 h1:dD3dhHNglpd98gs72my22Ndqi1hqQGllFFg1F+twfxg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25/go.mod h1:0yAbjPfd64gG7mj85RW+fMEYdfBgCRZw8g/oWcL1pjc= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDbiit/GcBE2K4IUpMZymaA0kOz3xK978= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298= +github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0= +github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 h1:1VwbP3qMNfxUDEXWki4rCE5iA+44VA1lokTz9HasGzw= +github.com/aws/aws-sdk-go-v2/service/signin v1.1.1/go.mod h1:vUtyoSj0OPji3kjIVSc/GlKuWEiL33f/WFxl6dmpy/A= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 h1:N6pIsdFOW1Kd9S4KyFKXdGRBojPPxkP32+uHFWLv4Hc= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.19/go.mod h1:3gt5WJArFooNmyLONS+h/R4J+o86II8du38IgCwj9dE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 h1:hc+lBYiiTr8Zk4MTzIsQ92MeDWCIDvWGmzKUWOaBcOg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2/go.mod h1:hU6fqB3OJA6/ePheD47LQnxvjYk6br6PtQxs+Q9ojvk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 h1:ErklX/7uhSbkAAeyQD/Y1OoQ9hO3SJXQNEgksORW3Js= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.3/go.mod h1:ULe4HCzfKPiR6R3HEurE3b1upEkuk8AkMrOKtaOxKO8= +github.com/aws/smithy-go v1.26.0 h1:9ouqbi+NyKP7fV3Te7UElCwdAb6Y8uk7LGwPE5tVe/s= +github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bkielbasa/cyclop v1.2.3 h1:faIVMIGDIANuGPWH031CZJTi2ymOQBULs9H21HSMa5w= +github.com/bkielbasa/cyclop v1.2.3/go.mod h1:kHTwA9Q0uZqOADdupvcFJQtp/ksSnytRMe8ztxG8Fuo= +github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= +github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/bombsimon/wsl/v4 v4.7.0 h1:1Ilm9JBPRczjyUs6hvOPKvd7VL1Q++PL8M0SXBDf+jQ= +github.com/bombsimon/wsl/v4 v4.7.0/go.mod h1:uV/+6BkffuzSAVYD+yGyld1AChO7/EuLrCF/8xTiapg= +github.com/bombsimon/wsl/v5 v5.6.0 h1:4z+/sBqC5vUmSp1O0mS+czxwH9+LKXtCWtHH9rZGQL8= +github.com/bombsimon/wsl/v5 v5.6.0/go.mod h1:Uqt2EfrMj2NV8UGoN1f1Y3m0NpUVCsUdrNCdet+8LvU= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/breml/bidichk v0.3.3 h1:WSM67ztRusf1sMoqH6/c4OBCUlRVTKq+CbSeo0R17sE= +github.com/breml/bidichk v0.3.3/go.mod h1:ISbsut8OnjB367j5NseXEGGgO/th206dVa427kR8YTE= +github.com/breml/errchkjson v0.4.1 h1:keFSS8D7A2T0haP9kzZTi7o26r7kE3vymjZNeNDRDwg= +github.com/breml/errchkjson v0.4.1/go.mod h1:a23OvR6Qvcl7DG/Z4o0el6BRAjKnaReoPQFciAl9U3s= +github.com/butuzov/ireturn v0.4.0 h1:+s76bF/PfeKEdbG8b54aCocxXmi0wvYdOVsWxVO7n8E= +github.com/butuzov/ireturn v0.4.0/go.mod h1:ghI0FrCmap8pDWZwfPisFD1vEc56VKH4NpQUxDHta70= +github.com/butuzov/mirror v1.3.0 h1:HdWCXzmwlQHdVhwvsfBb2Au0r3HyINry3bDWLYXiKoc= +github.com/butuzov/mirror v1.3.0/go.mod h1:AEij0Z8YMALaq4yQj9CPPVYOyJQyiexpQEQgihajRfI= +github.com/catenacyber/perfsprint v0.10.1 h1:u7Riei30bk46XsG8nknMhKLXG9BcXz3+3tl/WpKm0PQ= +github.com/catenacyber/perfsprint v0.10.1/go.mod h1:DJTGsi/Zufpuus6XPGJyKOTMELe347o6akPvWG9Zcsc= +github.com/ccojocar/zxcvbn-go v1.0.4 h1:FWnCIRMXPj43ukfX000kvBZvV6raSxakYr1nzyNrUcc= +github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das= github.com/cenk/backoff v2.2.1+incompatible h1:djdFT7f4gF2ttuzRKPbMOWgZajgesItGLwG5FTQKmmE= github.com/cenk/backoff v2.2.1+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charithe/durationcheck v0.0.11 h1:g1/EX1eIiKS57NTWsYtHDZ/APfeXKhye1DidBcABctk= +github.com/charithe/durationcheck v0.0.11/go.mod h1:x5iZaixRNl8ctbM+3B2RrPG5t856TxRyVQEnbIEM2X4= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.10.1 h1:rL3Koar5XvX0pHGfovN03f5cxLbCF2YvLeyz7D2jVDQ= +github.com/charmbracelet/x/ansi v0.10.1/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/ckaznocha/intrange v0.3.1 h1:j1onQyXvHUsPWujDH6WIjhyH26gkRt/txNlV7LspvJs= +github.com/ckaznocha/intrange v0.3.1/go.mod h1:QVepyz1AkUoFQkpEqksSYpNpUo3c5W7nWh/s6SHIJJk= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/curioswitch/go-reassign v0.3.0 h1:dh3kpQHuADL3cobV/sSGETA8DOv457dwl+fbBAhrQPs= +github.com/curioswitch/go-reassign v0.3.0/go.mod h1:nApPCCTtqLJN/s8HfItCcKV0jIPwluBOvZP+dsJGA88= +github.com/daixiang0/gci v0.13.7 h1:+0bG5eK9vlI08J+J/NWGbWPTNiXPG4WhNLJOkSxWITQ= +github.com/daixiang0/gci v0.13.7/go.mod h1:812WVN6JLFY9S6Tv76twqmNqevN0pa3SX3nih0brVzQ= +github.com/dave/dst v0.27.3 h1:P1HPoMza3cMEquVf9kKy8yXsFirry4zEnWOdYPOoIzY= +github.com/dave/dst v0.27.3/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEyc= +github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo= +github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= -github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= -github.com/ecosyste-ms/ecosystems-go v0.5.0 h1:qZqgar3Do9RrtW327OXlsrGKiRvNdeX8CmXb1aX4Njo= -github.com/ecosyste-ms/ecosystems-go v0.5.0/go.mod h1:IhsYBZfxkofxya0O1O2DlhDt281utWJ6VQO0mPVb99I= +github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= +github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ecosyste-ms/ecosystems-go v0.2.0 h1:Nhpg54C+St8Sd/mf8bNJmQqx35ZgHw33TfFoxaXMQI8= +github.com/ecosyste-ms/ecosystems-go v0.2.0/go.mod h1:CCdzT1iAZirbEZAbFSnWpK88eKKaIWex7gjtZ0UudXA= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= -github.com/envoyproxy/go-control-plane/envoy v1.39.0 h1:1uwRDYPYG8BIBU9Mj1sUAebNmlM6beu/ZKKweSLDxk8= -github.com/envoyproxy/go-control-plane/envoy v1.39.0/go.mod h1:5e4ylfTZO723MEEFsCpSW4ZEBWR8mwkEyXfwJBTCZ9c= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q= +github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= -github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= -github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/git-pkgs/archives v0.10.0 h1:/qgk0MI6VK2sKOoVTd0B4zs5hwTo+St0X8cYYftzumc= -github.com/git-pkgs/archives v0.10.0/go.mod h1:GJ/h7NPTUYiPinOUxpYNsHFCUikYZcJXvugBvP1VGNc= -github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx2xk= -github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= -github.com/git-pkgs/cooldown v0.3.1 h1:XDt54+ZuYoJclTLysaQ2GUZPhYYuJdjpaqeLSwjMdto= -github.com/git-pkgs/cooldown v0.3.1/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= -github.com/git-pkgs/enrichment v0.7.5 h1:jbfFzf7+HSS7el8AuMri4vHXq2izIIZDVIU0uYI5LFM= -github.com/git-pkgs/enrichment v0.7.5/go.mod h1:TbxC/nGQN96YCbdoiDs8a9z3gk2zXmLVS1989/jvnsU= -github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= -github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= -github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZOxM= -github.com/git-pkgs/integrity v0.2.0/go.mod h1:vPLb2RB+hR/iCMIgqph0PZbQ7iktXDTK+uupoHQikKk= -github.com/git-pkgs/magic v0.6.0 h1:fINPe67FEIaFRVlZzhOIDClMkm+gftBroBENa8bHSxQ= -github.com/git-pkgs/magic v0.6.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= -github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= -github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= -github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= -github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= -github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= -github.com/git-pkgs/sbom v0.1.9 h1:sfF5cJAwmgrm2N4HTXQWjiBe8tdu8xwhjwfAOyDNMfc= -github.com/git-pkgs/sbom v0.1.9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= -github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= -github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= -github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= -github.com/git-pkgs/vers v0.7.2/go.mod h1:ofLiBpPNkQmC0LB1k0zmN1gCv7Hi3MiZx8WtmWZ4A9A= -github.com/git-pkgs/vulns v0.2.4 h1:QnsQg9sWUfM8ZzA7A/1LSl8DJ/0/6Yrm7vfRwkpyXUY= -github.com/git-pkgs/vulns v0.2.4/go.mod h1:csd+UWvmYWVVAkka7FNbyz840/1R3QXS7LwD2Irsf4Q= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= +github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/firefart/nonamedreturns v1.0.6 h1:vmiBcKV/3EqKY3ZiPxCINmpS431OcE1S47AQUwhrg8E= +github.com/firefart/nonamedreturns v1.0.6/go.mod h1:R8NisJnSIpvPWheCq0mNRXJok6D8h7fagJTF8EMEwCo= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= +github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= +github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0= +github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= +github.com/git-pkgs/archives v0.3.0 h1:iXKyO83jEFub1PGEDlHmk2tQ7XeV5LySTc0sEkH3x78= +github.com/git-pkgs/archives v0.3.0/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU= +github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w= +github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= +github.com/git-pkgs/enrichment v0.4.1 h1:A8BKs0XwvpF1sF5qviZy4fkJAe18qB9OgpbRnmwnT34= +github.com/git-pkgs/enrichment v0.4.1/go.mod h1:stHqZUitV9ZkwACqHzBysLMSe6T4QZn81hxTdSroNhM= +github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= +github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= +github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= +github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= +github.com/git-pkgs/purl v0.1.13 h1:at8BU6vnP5oonHFHAPA064BzgRqij+SZcOUDgNT2DC8= +github.com/git-pkgs/purl v0.1.13/go.mod h1:8oCcdcYZA/e1B33e7Ylju6azboTKjdqf3ybcbQj6I/o= +github.com/git-pkgs/registries v0.6.2 h1:26G5zW6Q7x1CSfNkaEqEjRMJiA4JwfdKOCJ7Qm+u0a8= +github.com/git-pkgs/registries v0.6.2/go.mod h1:GR0Bu6nC3NQe6f7lfDoEVqAnoQkMocf4M98B12a7B3E= +github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= +github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= +github.com/git-pkgs/vers v0.2.6 h1:IelZd7BP/JhzTloUTDY67nehUgoYva3g9viqAMCHJg8= +github.com/git-pkgs/vers v0.2.6/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= +github.com/git-pkgs/vulns v0.1.6 h1:8RRSgdlxp4JMU0Zykr63XTOMo5CyZKwt/PwaQxrx9Yg= +github.com/git-pkgs/vulns v0.1.6/go.mod h1:TsZC4MjoCkKJslgmbcmRCnytwnFcjESC2N8b0a2xDWc= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= -github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg= -github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM= -github.com/go-jose/go-jose/v4 v4.1.5 h1:RjgjO2LOtWOJKUC5wpwY9LR3B3vwVAz6JS2YHfYU6eA= -github.com/go-jose/go-jose/v4 v4.1.5/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM= +github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= +github.com/go-critic/go-critic v0.14.3 h1:5R1qH2iFeo4I/RJU8vTezdqs08Egi4u5p6vOESA0pog= +github.com/go-critic/go-critic v0.14.3/go.mod h1:xwntfW6SYAd7h1OqDzmN6hBX/JxsEKl5up/Y2bsxgVQ= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= -github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -166,44 +287,143 @@ github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= -github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= +github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= +github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= +github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU= +github.com/go-toolsmith/astcopy v1.1.0 h1:YGwBN0WM+ekI/6SS6+52zLDEf8Yvp3n2seZITCUBt5s= +github.com/go-toolsmith/astcopy v1.1.0/go.mod h1:hXM6gan18VA1T/daUEHCFcYiW8Ai1tIwIzHY6srfEAw= +github.com/go-toolsmith/astequal v1.0.3/go.mod h1:9Ai4UglvtR+4up+bAD4+hCj7iTo4m/OXVTSLnCyTAx4= +github.com/go-toolsmith/astequal v1.1.0/go.mod h1:sedf7VIdCL22LD8qIvv7Nn9MuWJruQA/ysswh64lffQ= +github.com/go-toolsmith/astequal v1.2.0 h1:3Fs3CYZ1k9Vo4FzFhwwewC3CHISHDnVUPC4x0bI2+Cw= +github.com/go-toolsmith/astequal v1.2.0/go.mod h1:c8NZ3+kSFtFY/8lPso4v8LuJjdJiUFVnSuU3s0qrrDY= +github.com/go-toolsmith/astfmt v1.1.0 h1:iJVPDPp6/7AaeLJEruMsBUlOYCmvg0MoCfJprsOmcco= +github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlNMV634mhwuQ4= +github.com/go-toolsmith/astp v1.1.0 h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA= +github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA= +github.com/go-toolsmith/pkgload v1.2.2 h1:0CtmHq/02QhxcF7E9N5LIFcYFsMR5rdovfqTtRKkgIk= +github.com/go-toolsmith/pkgload v1.2.2/go.mod h1:R2hxLNRKuAsiXCo2i5J6ZQPhnPMOVtU+f0arbFPWCus= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw= +github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= +github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus= +github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-xmlfmt/xmlfmt v1.1.3 h1:t8Ey3Uy7jDSEisW2K3somuMKIpzktkWptA0iFCnRUWY= +github.com/go-xmlfmt/xmlfmt v1.1.3/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/godoc-lint/godoc-lint v0.11.2 h1:Bp0FkJWoSdNsBikdNgIcgtaoo+xz6I/Y9s5WSBQUeeM= +github.com/godoc-lint/godoc-lint v0.11.2/go.mod h1:iVpGdL1JCikNH2gGeAn3Hh+AgN5Gx/I/cxV+91L41jo= +github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= +github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golangci/asciicheck v0.5.0 h1:jczN/BorERZwK8oiFBOGvlGPknhvq0bjnysTj4nUfo0= +github.com/golangci/asciicheck v0.5.0/go.mod h1:5RMNAInbNFw2krqN6ibBxN/zfRFa9S6tA1nPdM0l8qQ= +github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 h1:WUvBfQL6EW/40l6OmeSBYQJNSif4O11+bmWEz+C7FYw= +github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32/go.mod h1:NUw9Zr2Sy7+HxzdjIULge71wI6yEg1lWQr7Evcu8K0E= +github.com/golangci/go-printf-func-name v0.1.1 h1:hIYTFJqAGp1iwoIfsNTpoq1xZAarogrvjO9AfiW3B4U= +github.com/golangci/go-printf-func-name v0.1.1/go.mod h1:Es64MpWEZbh0UBtTAICOZiB+miW53w/K9Or/4QogJss= +github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d h1:viFft9sS/dxoYY0aiOTsLKO2aZQAPT4nlQCsimGcSGE= +github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d/go.mod h1:ivJ9QDg0XucIkmwhzCDsqcnxxlDStoTl89jDMIoNxKY= +github.com/golangci/golangci-lint/v2 v2.10.1 h1:flhw5Px6ojbLyEFzXvJn5B2HEdkkRlkhE1SnmCbQBiE= +github.com/golangci/golangci-lint/v2 v2.10.1/go.mod h1:dBsrOk6zj0vDhlTv+IiJGqkDokR24IVTS7W3EVfPTQY= +github.com/golangci/golines v0.15.0 h1:Qnph25g8Y1c5fdo1X7GaRDGgnMHgnxh4Gk4VfPTtRx0= +github.com/golangci/golines v0.15.0/go.mod h1:AZjXd23tbHMpowhtnGlj9KCNsysj72aeZVVHnVcZx10= +github.com/golangci/misspell v0.8.0 h1:qvxQhiE2/5z+BVRo1kwYA8yGz+lOlu5Jfvtx2b04Jbg= +github.com/golangci/misspell v0.8.0/go.mod h1:WZyyI2P3hxPY2UVHs3cS8YcllAeyfquQcKfdeE9AFVg= +github.com/golangci/plugin-module-register v0.1.2 h1:e5WM6PO6NIAEcij3B053CohVp3HIYbzSuP53UAYgOpg= +github.com/golangci/plugin-module-register v0.1.2/go.mod h1:1+QGTsKBvAIvPvoY/os+G5eoqxWn70HYDm2uvUyGuVw= +github.com/golangci/revgrep v0.8.0 h1:EZBctwbVd0aMeRnNUsFogoyayvKHyxlV3CdUA46FX2s= +github.com/golangci/revgrep v0.8.0/go.mod h1:U4R/s9dlXZsg8uJmaR1GrloUr14D7qDl8gi2iPXJH8k= +github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e h1:ai0EfmVYE2bRA5htgAG9r7s3tHsfjIhN98WshBTJ9jM= +github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e/go.mod h1:Vrn4B5oR9qRwM+f54koyeH3yzphlecwERs0el27Fr/s= +github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e h1:gD6P7NEo7Eqtt0ssnqSJNNndxe69DOQ24A5h7+i3KpM= +github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e/go.mod h1:h+wZwLjUTJnm/P2rwlbJdRPZXOzaT36/FwnPnY2inzc= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-replayers/grpcreplay v1.5.1 h1:R90FMYaA6GQNhi/GG0D4zS14HEQ+wpO8bQAV0Qwyt08= -github.com/google/go-replayers/grpcreplay v1.5.1/go.mod h1:42YCe31fkUrhRo+haVAfLse+gNqGI1iO1wdNR+sWY9U= +github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo= +github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI= github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs= -github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= -github.com/googleapis/enterprise-certificate-proxy v0.3.21 h1:OFdQ3tnCX/zaQ0Cedur3D3z7kI6HiLX9g3TiAN4/DFU= -github.com/googleapis/enterprise-certificate-proxy v0.3.21/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= -github.com/googleapis/gax-go/v2 v2.24.1 h1:AtqTN21IXMMWo99LiEVAiBfNNQmO40d8xUfZI640mc0= -github.com/googleapis/gax-go/v2 v2.24.1/go.mod h1:bWeBei0NVwaNZKb2y1HUBS7gLXIF3/Tu3pq7j8D2Tb0= +github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8= +github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= +github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE= +github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA= +github.com/gordonklaus/ineffassign v0.2.0 h1:Uths4KnmwxNJNzq87fwQQDDnbNb7De00VOk9Nu0TySs= +github.com/gordonklaus/ineffassign v0.2.0/go.mod h1:TIpymnagPSexySzs7F9FnO1XFTy8IT3a59vmZp5Y9Lw= +github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk= +github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc= +github.com/gostaticanalysis/comment v1.4.2/go.mod h1:KLUTGDv6HOCotCH8h2erHKmpci2ZoR8VPu34YA2uzdM= +github.com/gostaticanalysis/comment v1.5.0 h1:X82FLl+TswsUMpMh17srGRuKaaXprTaytmEpgnKIDu8= +github.com/gostaticanalysis/comment v1.5.0/go.mod h1:V6eb3gpCv9GNVqb6amXzEUX3jXLVK/AdA+IrAMSqvEc= +github.com/gostaticanalysis/forcetypeassert v0.2.0 h1:uSnWrrUEYDr86OCxWa4/Tp2jeYDlogZiZHzGkWFefTk= +github.com/gostaticanalysis/forcetypeassert v0.2.0/go.mod h1:M5iPavzE9pPqWyeiVXSFghQjljW1+l/Uke3PXHS6ILY= +github.com/gostaticanalysis/nilerr v0.1.2 h1:S6nk8a9N8g062nsx63kUkF6AzbHGw7zzyHMcpu52xQU= +github.com/gostaticanalysis/nilerr v0.1.2/go.mod h1:A19UHhoY3y8ahoL7YKz6sdjDtduwTSI4CsymaC2htPA= +github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= +github.com/gostaticanalysis/testutil v0.5.0 h1:Dq4wT1DdTwTGCQQv3rl3IvD5Ld0E6HiY+3Zh0sUGqw8= +github.com/gostaticanalysis/testutil v0.5.0/go.mod h1:OLQSbuM6zw2EvCcXTz1lVq5unyoNft372msDY0nY5Hs= +github.com/hashicorp/go-immutable-radix/v2 v2.1.0 h1:CUW5RYIcysz+D3B+l1mDeXrQ7fUvGGCwJfdASSzbrfo= +github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= +github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jgautheron/goconst v1.8.2 h1:y0XF7X8CikZ93fSNT6WBTb/NElBu9IjaY7CCYQrCMX4= +github.com/jgautheron/goconst v1.8.2/go.mod h1:A0oxgBCHy55NQn6sYpO7UdnA9p+h7cPtoOZUmvNIako= +github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs= +github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c= +github.com/jjti/go-spancheck v0.6.5 h1:lmi7pKxa37oKYIMScialXUK6hP3iY5F1gu+mLBPgYB8= +github.com/jjti/go-spancheck v0.6.5/go.mod h1:aEogkeatBrbYsyW6y5TgDfihCulDYciL1B7rG2vSsrU= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/julz/importas v0.2.0 h1:y+MJN/UdL63QbFJHws9BVC5RpA2iq0kpjrFajTGivjQ= +github.com/julz/importas v0.2.0/go.mod h1:pThlt589EnCYtMnmhmRYY/qn9lCf/frPOK+WMx3xiJY= +github.com/karamaru-alpha/copyloopvar v1.2.2 h1:yfNQvP9YaGQR7VaWLYcfZUlRP2eo2vhExWKxD/fP6q0= +github.com/karamaru-alpha/copyloopvar v1.2.2/go.mod h1:oY4rGZqZ879JkJMtX3RRkcXRkmUvH0x35ykgaKgsgJY= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= -github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3M= +github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= +github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= +github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -211,182 +431,488 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kulti/thelper v0.7.1 h1:fI8QITAoFVLx+y+vSyuLBP+rcVIB8jKooNSCT2EiI98= +github.com/kulti/thelper v0.7.1/go.mod h1:NsMjfQEy6sd+9Kfw8kCP61W1I0nerGSYSFnGaxQkcbs= +github.com/kunwardeep/paralleltest v1.0.15 h1:ZMk4Qt306tHIgKISHWFJAO1IDQJLc6uDyJMLyncOb6w= +github.com/kunwardeep/paralleltest v1.0.15/go.mod h1:di4moFqtfz3ToSKxhNjhOZL+696QtJGCFe132CbBLGk= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lasiar/canonicalheader v1.1.2 h1:vZ5uqwvDbyJCnMhmFYimgMZnJMjwljN5VGY0VKbMXb4= +github.com/lasiar/canonicalheader v1.1.2/go.mod h1:qJCeLFS0G/QlLQ506T+Fk/fWMa2VmBUiEI2cuMK4djI= +github.com/ldez/exptostd v0.4.5 h1:kv2ZGUVI6VwRfp/+bcQ6Nbx0ghFWcGIKInkG/oFn1aQ= +github.com/ldez/exptostd v0.4.5/go.mod h1:QRjHRMXJrCTIm9WxVNH6VW7oN7KrGSht69bIRwvdFsM= +github.com/ldez/gomoddirectives v0.8.0 h1:JqIuTtgvFC2RdH1s357vrE23WJF2cpDCPFgA/TWDGpk= +github.com/ldez/gomoddirectives v0.8.0/go.mod h1:jutzamvZR4XYJLr0d5Honycp4Gy6GEg2mS9+2YX3F1Q= +github.com/ldez/grignotin v0.10.1 h1:keYi9rYsgbvqAZGI1liek5c+jv9UUjbvdj3Tbn5fn4o= +github.com/ldez/grignotin v0.10.1/go.mod h1:UlDbXFCARrXbWGNGP3S5vsysNXAPhnSuBufpTEbwOas= +github.com/ldez/structtags v0.6.1 h1:bUooFLbXx41tW8SvkfwfFkkjPYvFFs59AAMgVg6DUBk= +github.com/ldez/structtags v0.6.1/go.mod h1:YDxVSgDy/MON6ariaxLF2X09bh19qL7MtGBN5MrvbdY= +github.com/ldez/tagliatelle v0.7.2 h1:KuOlL70/fu9paxuxbeqlicJnCspCRjH0x8FW+NfgYUk= +github.com/ldez/tagliatelle v0.7.2/go.mod h1:PtGgm163ZplJfZMZ2sf5nhUT170rSuPgBimoyYtdaSI= +github.com/ldez/usetesting v0.5.0 h1:3/QtzZObBKLy1F4F8jLuKJiKBjjVFi1IavpoWbmqLwc= +github.com/ldez/usetesting v0.5.0/go.mod h1:Spnb4Qppf8JTuRgblLrEWb7IE6rDmUpGvxY3iRrzvDQ= +github.com/leonklingele/grouper v1.1.2 h1:o1ARBDLOmmasUaNDesWqWCIFH3u7hoFlM84YrjT3mIY= +github.com/leonklingele/grouper v1.1.2/go.mod h1:6D0M/HVkhs2yRKRFZUoGjeDy7EZTfFBE9gl4kjmIGkA= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/macabu/inamedparam v0.2.0 h1:VyPYpOc10nkhI2qeNUdh3Zket4fcZjEWe35poddBCpE= +github.com/macabu/inamedparam v0.2.0/go.mod h1:+Pee9/YfGe5LJ62pYXqB89lJ+0k5bsR8Wgz/C0Zlq3U= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= +github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= -github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/manuelarte/embeddedstructfieldcheck v0.4.0 h1:3mAIyaGRtjK6EO9E73JlXLtiy7ha80b2ZVGyacxgfww= +github.com/manuelarte/embeddedstructfieldcheck v0.4.0/go.mod h1:z8dFSyXqp+fC6NLDSljRJeNQJJDWnY7RoWFzV3PC6UM= +github.com/manuelarte/funcorder v0.5.0 h1:llMuHXXbg7tD0i/LNw8vGnkDTHFpTnWqKPI85Rknc+8= +github.com/manuelarte/funcorder v0.5.0/go.mod h1:Yt3CiUQthSBMBxjShjdXMexmzpP8YGvGLjrxJNkO2hA= +github.com/maratori/testableexamples v1.0.1 h1:HfOQXs+XgfeRBJ+Wz0XfH+FHnoY9TVqL6Fcevpzy4q8= +github.com/maratori/testableexamples v1.0.1/go.mod h1:XE2F/nQs7B9N08JgyRmdGjYVGqxWwClLPCGSQhXQSrQ= +github.com/maratori/testpackage v1.1.2 h1:ffDSh+AgqluCLMXhM19f/cpvQAKygKAJXFl9aUjmbqs= +github.com/maratori/testpackage v1.1.2/go.mod h1:8F24GdVDFW5Ew43Et02jamrVMNXLUNaOynhDssITGfc= +github.com/matoous/godox v1.1.0 h1:W5mqwbyWrwZv6OQ5Z1a/DHGMOvXYCBP3+Ht7KMoJhq4= +github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= +github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mgechev/revive v1.14.0 h1:CC2Ulb3kV7JFYt+izwORoS3VT/+Plb8BvslI/l1yZsc= +github.com/mgechev/revive v1.14.0/go.mod h1:MvnujelCZBZCaoDv5B3foPo6WWgULSSFxvfxp7GsPfo= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moricho/tparallel v0.3.2 h1:odr8aZVFA3NZrNybggMkYO3rgPRcqjeQUlBBFVxKHTI= +github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk= -github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g= +github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U= +github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oapi-codegen/nullable v1.2.0 h1:VflFkDW980KhBPiFF7nWSyjg+r4Obqj8lXipV0UkP5w= -github.com/oapi-codegen/nullable v1.2.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= -github.com/oapi-codegen/runtime v1.7.0 h1:t7358VYPvNbWJ9gdAkIK/smVeHpBf6yp8VTsaZsb/7k= -github.com/oapi-codegen/runtime v1.7.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/package-url/packageurl-go v0.1.7 h1:iFWg6tzAjLA6F/qX3M5nZaiMHJgc+p2zxVyr/fY+sZY= -github.com/package-url/packageurl-go v0.1.7/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= -github.com/pandatix/go-cvss v0.6.4 h1:9w2RCO/Q4UTiJyEgpCHRiVc6CfrsFEnkoX+OtATqKio= -github.com/pandatix/go-cvss v0.6.4/go.mod h1:/ukvQnYlrKl3o/DVp7/GO2UZyZheuo/maOK0U1nBEhQ= +github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg= +github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= +github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= +github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= +github.com/nunnatsa/ginkgolinter v0.23.0 h1:x3o4DGYOWbBMP/VdNQKgSj+25aJKx2Pe6lHr8gBcgf8= +github.com/nunnatsa/ginkgolinter v0.23.0/go.mod h1:9qN1+0akwXEccwV1CAcCDfcoBlWXHB+ML9884pL4SZ4= +github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= +github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= +github.com/oapi-codegen/runtime v1.4.1 h1:9nwLoI+KrWxzbBcp0jO/R8uXqbik/HUyCvPeU68Y/qo= +github.com/oapi-codegen/runtime v1.4.1/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= +github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= +github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= +github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= +github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/package-url/packageurl-go v0.1.6 h1:YO3p6u1XmCUliivUg/qWphaY8vI6hxSnnPv7Bfg3m5M= +github.com/package-url/packageurl-go v0.1.6/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= +github.com/pandatix/go-cvss v0.6.2 h1:TFiHlzUkT67s6UkelHmK6s1INKVUG7nlKYiWWDTITGI= +github.com/pandatix/go-cvss v0.6.2/go.mod h1:jDXYlQBZrc8nvrMUVVvTG8PhmuShOnKrxP53nOFkt8Q= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea h1:sKwxy1H95npauwu8vtF95vG/syrL0p8fSZo/XlDg5gk= github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea/go.mod h1:1VcHEd3ro4QMoHfiNl/j7Jkln9+KQuorp0PItHMJYNg= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0= -github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk= -github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= -github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= -github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4= -github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk= -github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics= -github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/quasilyte/go-ruleguard v0.4.5 h1:AGY0tiOT5hJX9BTdx/xBdoCubQUAE2grkqY2lSwvZcA= +github.com/quasilyte/go-ruleguard v0.4.5/go.mod h1:Vl05zJ538vcEEwu16V/Hdu7IYZWyKSwIy4c88Ro1kRE= +github.com/quasilyte/go-ruleguard/dsl v0.3.23 h1:lxjt5B6ZCiBeeNO8/oQsegE6fLeCzuMRoVWSkXC4uvY= +github.com/quasilyte/go-ruleguard/dsl v0.3.23/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= +github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= +github.com/quasilyte/gogrep v0.5.0/go.mod h1:Cm9lpz9NZjEoL1tgZ2OgeUKPIxL1meE7eo60Z6Sk+Ng= +github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl980XxGFEZSS6KlBGIV0diGdySzxATTWoqaU= +github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= +github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= +github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= +github.com/raeperd/recvcheck v0.2.0 h1:GnU+NsbiCqdC2XX5+vMZzP+jAJC5fht7rcVTAhX74UI= +github.com/raeperd/recvcheck v0.2.0/go.mod h1:n04eYkwIR0JbgD73wT8wL4JjPC3wm0nFtzBnWNocnYU= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= -github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 h1:18kd+8ZUlt/ARXhljq+14TwAoKa61q6dX8jtwOf6DH8= github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6rVCAYOp8+kHUyFvL6Mhk= github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= -github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= -github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryancurrah/gomodguard v1.4.1 h1:eWC8eUMNZ/wM/PWuZBv7JxxqT5fiIKSIyTvjb7Elr+g= +github.com/ryancurrah/gomodguard v1.4.1/go.mod h1:qnMJwV1hX9m+YJseXEBhd2s90+1Xn6x9dLz11ualI1I= +github.com/ryanrolds/sqlclosecheck v0.5.1 h1:dibWW826u0P8jNLsLN+En7+RqWWTYrjCB9fJfSfdyCU= +github.com/ryanrolds/sqlclosecheck v0.5.1/go.mod h1:2g3dUjoS6AL4huFdv6wn55WpLIDjY7ZgUR4J8HOO/XQ= +github.com/sanposhiho/wastedassign/v2 v2.1.0 h1:crurBF7fJKIORrV85u9UUpePDYGWnwvv3+A96WvwXT0= +github.com/sanposhiho/wastedassign/v2 v2.1.0/go.mod h1:+oSmSC+9bQ+VUAxA66nBb0Z7N8CK7mscKTDYC6aIek4= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tMEOsumirXcOJqAw= +github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= +github.com/sashamelentyev/usestdlibvars v1.29.0 h1:8J0MoRrw4/NAXtjQqTHrbW9NN+3iMf7Knkq057v4XOQ= +github.com/sashamelentyev/usestdlibvars v1.29.0/go.mod h1:8PpnjHMk5VdeWlVb4wCdrB8PNbLqZ3wBZTZWkrpZZL8= +github.com/securego/gosec/v2 v2.23.0 h1:h4TtF64qFzvnkqvsHC/knT7YC5fqyOCItlVR8+ptEBo= +github.com/securego/gosec/v2 v2.23.0/go.mod h1:qRHEgXLFuYUDkI2T7W7NJAmOkxVhkR0x9xyHOIcMNZ0= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+Wwfd0XE= +github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4= +github.com/sonatard/noctx v0.4.0 h1:7MC/5Gg4SQ4lhLYR6mvOP6mQVSxCrdyiExo7atBs27o= +github.com/sonatard/noctx v0.4.0/go.mod h1:64XdbzFb18XL4LporKXp8poqZtPKbCrqQ402CV+kJas= +github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0= +github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= +github.com/spdx/tools-golang v0.5.7 h1:+sWcKGnhwp3vLdMqPcLdA6QK679vd86cK9hQWH3AwCg= +github.com/spdx/tools-golang v0.5.7/go.mod h1:jg7w0LOpoNAw6OxKEzCoqPC2GCTj45LyTlVmXubDsYw= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= +github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= +github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= +github.com/stbenjam/no-sprintf-host-port v0.3.1 h1:AyX7+dxI4IdLBPtDbsGAyqiTSLpCP9hWRrXQDU4Cm/g= +github.com/stbenjam/no-sprintf-host-port v0.3.1/go.mod h1:ODbZesTCHMVKthBHskvUUexdcNHAQRXk9NpSsL8p/HQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= -github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= -github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4= -github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= -github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= -github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= -github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= -github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/tenntenn/modver v1.0.1 h1:2klLppGhDgzJrScMpkj9Ujy3rXPUspSjAcev9tSEBgA= +github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= +github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpRQGxTSkNYKJ51yaw6ChIqO+Je8UqsTKN/cDag= +github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= +github.com/terminalstatic/go-xsd-validate v0.1.6 h1:TenYeQ3eY631qNi1/cTmLH/s2slHPRKTTHT+XSHkepo= +github.com/terminalstatic/go-xsd-validate v0.1.6/go.mod h1:18lsvYFofBflqCrvo1umpABZ99+GneNTw2kEEc8UPJw= +github.com/tetafro/godot v1.5.4 h1:u1ww+gqpRLiIA16yF2PV1CV1n/X3zhyezbNXC3E14Sg= +github.com/tetafro/godot v1.5.4/go.mod h1:eOkMrVQurDui411nBY2FA05EYH01r14LuWY/NrVDVcU= +github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 h1:9LPGD+jzxMlnk5r6+hJnar67cgpDIz/iyD+rfl5r2Vk= +github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= +github.com/timonwong/loggercheck v0.11.0 h1:jdaMpYBl+Uq9mWPXv1r8jc5fC3gyXx4/WGwTnnNKn4M= +github.com/timonwong/loggercheck v0.11.0/go.mod h1:HEAWU8djynujaAVX7QI65Myb8qgfcZ1uKbdpg3ZzKl8= +github.com/tomarrell/wrapcheck/v2 v2.12.0 h1:H/qQ1aNWz/eeIhxKAFvkfIA+N7YDvq6TWVFL27Of9is= +github.com/tomarrell/wrapcheck/v2 v2.12.0/go.mod h1:AQhQuZd0p7b6rfW+vUwHm5OMCGgp63moQ9Qr/0BpIWo= +github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw= +github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLkI= +github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA= +github.com/ultraware/whitespace v0.2.0 h1:TYowo2m9Nfj1baEQBjuHzvMRbp19i+RCcRYrSWoFa+g= +github.com/ultraware/whitespace v0.2.0/go.mod h1:XcP1RLD81eV4BW8UhQlpaR+SDc2givTvyI8a586WjW8= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/uudashr/gocognit v1.2.0 h1:3BU9aMr1xbhPlvJLSydKwdLN3tEUUrzPSSM8S4hDYRA= +github.com/uudashr/gocognit v1.2.0/go.mod h1:k/DdKPI6XBZO1q7HgoV2juESI2/Ofj9AcHPZhBBdrTU= +github.com/uudashr/iface v1.4.1 h1:J16Xl1wyNX9ofhpHmQ9h9gk5rnv2A6lX/2+APLTo0zU= +github.com/uudashr/iface v1.4.1/go.mod h1:pbeBPlbuU2qkNDn0mmfrxP2X+wjPMIQAy+r1MBXSXtg= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xen0n/gosmopolitan v1.3.0 h1:zAZI1zefvo7gcpbCOrPSHJZJYA9ZgLfJqtKzZ5pHqQM= +github.com/xen0n/gosmopolitan v1.3.0/go.mod h1:rckfr5T6o4lBtM1ga7mLGKZmLxswUoH1zxHgNXOsEt4= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM= +github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= +github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs= +github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4= +github.com/ykadowak/zerologlint v0.1.5 h1:Gy/fMz1dFQN9JZTPjv1hxEk+sRWm05row04Yoolgdiw= +github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +gitlab.com/bosi/decorder v0.4.2 h1:qbQaV3zgwnBZ4zPMhGLW4KZe7A7NwxEhJx39R3shffo= +gitlab.com/bosi/decorder v0.4.2/go.mod h1:muuhHoaJkA9QLcYHq4Mj8FJUwDZ+EirSHRiaTcTf6T8= +go-simpler.org/assert v0.9.0 h1:PfpmcSvL7yAnWyChSjOz6Sp6m9j5lyK8Ok9pEL31YkQ= +go-simpler.org/assert v0.9.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28= +go-simpler.org/musttag v0.14.0 h1:XGySZATqQYSEV3/YTy+iX+aofbZZllJaqwFWs+RTtSo= +go-simpler.org/musttag v0.14.0/go.mod h1:uP8EymctQjJ4Z1kUnjX0u2l60WfUdQxCwSNKzE1JEOE= +go-simpler.org/sloglint v0.11.1 h1:xRbPepLT/MHPTCA6TS/wNfZrDzkGvCCqUv4Bdwc3H7s= +go-simpler.org/sloglint v0.11.1/go.mod h1:2PowwiCOK8mjiF+0KGifVOT8ZsCNiFzvfyJeJOIt8MQ= +go.augendre.info/arangolint v0.4.0 h1:xSCZjRoS93nXazBSg5d0OGCi9APPLNMmmLrC995tR50= +go.augendre.info/arangolint v0.4.0/go.mod h1:l+f/b4plABuFISuKnTGD4RioXiCCgghv2xqst/xOvAA= +go.augendre.info/fatcontext v0.9.0 h1:Gt5jGD4Zcj8CDMVzjOJITlSb9cEch54hjRRlN3qDojE= +go.augendre.info/fatcontext v0.9.0/go.mod h1:L94brOAT1OOUNue6ph/2HnwxoNlds9aXDF2FcUntbNw= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.46.0 h1:PI8dGkqDaQkwJ8kOopqMhDTbrnK3UIeG/RCHH4HErbo= -go.opentelemetry.io/contrib/detectors/gcp v1.46.0/go.mod h1:nsrN5c/sOLoY2vsPxN/rQ0V0nvGrWJCqcW4UXLtqNG8= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= -go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= -go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= -go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= -go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= -go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU= -go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4= -go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= -go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= -go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= -go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= -go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= -go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= -go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= -go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -gocloud.dev v0.47.0 h1:qN4/0vtutn97f1X5ziisf3WXuaFL9AR0FIRnyJRYzgA= -gocloud.dev v0.47.0/go.mod h1:OdMnyXQRhM17U0a0DwU57pe0c01qN3lJNmaeeRYW0D0= -golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= -golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= -golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= -golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= +gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= +golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= +golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 h1:qWFG1Dj7TBjOjOvhEOkmyGPVoquqUKnIU0lEVLp8xyk= +golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358/go.mod h1:4Mzdyp/6jzw9auFDJ3OMF5qksa7UvPnzKqTVGcb04ms= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= -golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= -golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= -golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= +golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= -golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= -golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= -golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= -golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= +golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.297.0 h1:WktxTsnnx0yZNnsR6j0q6hR21RnnK81FHTOPy/ux4OE= -google.golang.org/api v0.297.0/go.mod h1:S4m8x0M6OkQpkOzGk1y9JG2sm4fFQrMh6dxzjCTszhE= -google.golang.org/genproto v0.0.0-20260904194346-d0f1323225a4 h1:6QRNo34FxJcE1LG6XfnmaQ17m6w21oXaKlKoBaF+UqE= -google.golang.org/genproto v0.0.0-20260904194346-d0f1323225a4/go.mod h1:eifJ91nefXnfP/bMkZsACVGm14ZtEL4eUXRLTryfbY4= -google.golang.org/genproto/googleapis/api v0.0.0-20260904194346-d0f1323225a4 h1:NCe/UiklGd/9xjT+ROBVhJ1kf6TRQaFedsR+z7u1gvo= -google.golang.org/genproto/googleapis/api v0.0.0-20260904194346-d0f1323225a4/go.mod h1:fJ2lYaWjqNknJyQBOCd0fA3HnEElJqGplH71a2txi+g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4 h1:5t+ZydAFj5kGVLrgCvLmpmCf9ylGRd64hpEronfRaws= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= -google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= -google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= -google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= -google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/api v0.272.0 h1:eLUQZGnAS3OHn31URRf9sAmRk3w2JjMx37d2k8AjJmA= +google.golang.org/api v0.272.0/go.mod h1:wKjowi5LNJc5qarNvDCvNQBn3rVK8nSy6jg2SwRwzIA= +google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE= +google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw= +google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js= +google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 h1:aJmi6DVGGIStN9Mobk/tZOOQUBbj0BPjZjjnOdoZKts= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/cc/v4 v4.30.3 h1:aDq8gIX1X7+VFgO/JYYji6pLWL2H4tv3g8WatT3KCys= -modernc.org/cc/v4 v4.30.3/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.37.0 h1:BAuJCUtIHIxFIYaFhqXIw5YaoMBlNCUlFovejHfMip4= -modernc.org/ccgo/v4 v4.37.0/go.mod h1:/8627VhwWtAbqUXmPORxFQU5SgLrAx2jl9seLw9GBhY= +honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= +honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= +modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= +modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= +modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.2.0 h1:/Y3LGDMQaoYf55PebP1qsZ5FDGgAZyG1RYKEHeQzh1o= -modernc.org/gc/v3 v3.2.0/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= +modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8= -modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= -modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.61.0 h1:sQi63oLQ9uquAUA4CzOisH35Q3pEszkPBuHM0tFL8CY= -modernc.org/sqlite v1.61.0/go.mod h1:f4gr5tbpx2umklaT/QVtbi/50hNdM+4ng+2zMo8KXJ0= +modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= +modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +mvdan.cc/gofumpt v0.9.2 h1:zsEMWL8SVKGHNztrx6uZrXdp7AX8r421Vvp23sz7ik4= +mvdan.cc/gofumpt v0.9.2/go.mod h1:iB7Hn+ai8lPvofHd9ZFGVg2GOr8sBUw1QUWjNbmIL/s= +mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 h1:ssMzja7PDPJV8FStj7hq9IKiuiKhgz9ErWw+m68e7DI= +mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15/go.mod h1:4M5MMXl2kW6fivUT6yRGpLLPNfuGtU2Z0cPvFquGDYU= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/accesslog/accesslog.go b/internal/accesslog/accesslog.go deleted file mode 100644 index 9ff4a0a..0000000 --- a/internal/accesslog/accesslog.go +++ /dev/null @@ -1,120 +0,0 @@ -// Package accesslog writes proxy activity as JSON Lines. -package accesslog - -import ( - "context" - "encoding/json" - "fmt" - "net/url" - "os" - "sync" - "time" -) - -const ( - accessLogFileMode os.FileMode = 0o600 - - // EventRequest identifies the response sent by the proxy to a client. - EventRequest = "request" - // EventUpstream identifies one HTTP exchange with an upstream service. - EventUpstream = "upstream" -) - -type requestIDKey struct{} - -// Entry is one proxy activity record. -type Entry struct { - Time time.Time `json:"time"` - Event string `json:"event"` - RequestID string `json:"request_id,omitempty"` - Method string `json:"method"` - Path string `json:"path,omitempty"` - URL string `json:"url,omitempty"` - StatusCode int `json:"status_code,omitempty"` - DurationMS int64 `json:"duration_ms"` - RemoteAddr string `json:"remote_addr,omitempty"` - // RemoteIP is the address the request is attributed to: the TCP peer, or - // the leftmost X-Forwarded-For entry when that header is trusted. - RemoteIP string `json:"remote_ip,omitempty"` - // UserAgent is recorded verbatim; Client is it reduced to a known tool - // name, so log queries can group without parsing. - UserAgent string `json:"user_agent,omitempty"` - Client string `json:"client,omitempty"` - Ecosystem string `json:"ecosystem,omitempty"` - // Bytes is always emitted, including as 0, so a log pipeline summing the - // field does not have to treat a bodyless response as null. - Bytes int64 `json:"bytes"` - Error string `json:"error,omitempty"` -} - -// Logger appends complete JSON objects to a file, one per line. -type Logger struct { - mu sync.Mutex - file *os.File - encoder *json.Encoder -} - -// Open opens path for append, creating it with owner-only permissions when needed. -func Open(path string) (*Logger, error) { - file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, accessLogFileMode) - if err != nil { - return nil, fmt.Errorf("opening access log: %w", err) - } - - return &Logger{ - file: file, - encoder: json.NewEncoder(file), - }, nil -} - -// Write appends an entry to the log. -func (l *Logger) Write(entry Entry) error { - if entry.Time.IsZero() { - entry.Time = time.Now().UTC() - } - - l.mu.Lock() - defer l.mu.Unlock() - - if err := l.encoder.Encode(entry); err != nil { - return fmt.Errorf("writing access log: %w", err) - } - return nil -} - -// Close closes the log file after any active writer finishes. -func (l *Logger) Close() error { - l.mu.Lock() - defer l.mu.Unlock() - - if err := l.file.Close(); err != nil { - return fmt.Errorf("closing access log: %w", err) - } - return nil -} - -// WithRequestID stores a proxy request ID in ctx. -func WithRequestID(ctx context.Context, requestID string) context.Context { - return context.WithValue(ctx, requestIDKey{}, requestID) -} - -// RequestID returns the proxy request ID stored in ctx. -func RequestID(ctx context.Context) string { - requestID, _ := ctx.Value(requestIDKey{}).(string) - return requestID -} - -// URLWithoutSecrets returns a URL without user information, query values, or fragments. -func URLWithoutSecrets(value *url.URL) string { - if value == nil { - return "" - } - - clean := *value - clean.User = nil - clean.RawQuery = "" - clean.ForceQuery = false - clean.Fragment = "" - clean.RawFragment = "" - return clean.String() -} diff --git a/internal/accesslog/accesslog_test.go b/internal/accesslog/accesslog_test.go deleted file mode 100644 index 4e53fa8..0000000 --- a/internal/accesslog/accesslog_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package accesslog - -import ( - "bufio" - "context" - "encoding/json" - "net/url" - "os" - "path/filepath" - "sync" - "testing" -) - -func TestLoggerWritesJSONLines(t *testing.T) { - path := filepath.Join(t.TempDir(), "access.jsonl") - logger, err := Open(path) - if err != nil { - t.Fatal(err) - } - - const entries = 20 - var wg sync.WaitGroup - for range entries { - wg.Add(1) - go func() { - defer wg.Done() - if err := logger.Write(Entry{ - Event: EventUpstream, - RequestID: "request-id", - Method: "GET", - URL: "https://registry.example/packages/example", - StatusCode: 429, - }); err != nil { - t.Errorf("Write: %v", err) - } - }() - } - wg.Wait() - - if err := logger.Close(); err != nil { - t.Fatal(err) - } - - file, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer func() { _ = file.Close() }() - - scanner := bufio.NewScanner(file) - count := 0 - for scanner.Scan() { - var entry Entry - if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil { - t.Fatalf("line %d is not JSON: %v", count+1, err) - } - if entry.Time.IsZero() { - t.Errorf("line %d has no time", count+1) - } - if entry.StatusCode != 429 { - t.Errorf("line %d status_code = %d, want 429", count+1, entry.StatusCode) - } - count++ - } - if err := scanner.Err(); err != nil { - t.Fatal(err) - } - if count != entries { - t.Errorf("lines = %d, want %d", count, entries) - } -} - -func TestRequestID(t *testing.T) { - ctx := WithRequestID(context.Background(), "abc-123") - if got := RequestID(ctx); got != "abc-123" { - t.Errorf("RequestID = %q, want %q", got, "abc-123") - } -} - -func TestURLWithoutSecrets(t *testing.T) { - value, err := url.Parse("https://user:password@registry.example/package.tgz?token=secret#fragment") - if err != nil { - t.Fatal(err) - } - - got := URLWithoutSecrets(value) - want := "https://registry.example/package.tgz" - if got != want { - t.Errorf("URLWithoutSecrets = %q, want %q", got, want) - } -} diff --git a/internal/config/config.go b/internal/config/config.go index 25536e7..2d275c7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,22 +24,10 @@ // storage: // url: "s3://bucket?endpoint=http://localhost:9000" // -// Google Cloud Storage: -// -// storage: -// url: "gs://bucket-name" -// // For S3, configure credentials via AWS environment variables: // // AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION // -// For GCS, authentication uses Application Default Credentials. This supports -// GKE Workload Identity, attached service accounts on GCE and Cloud Run, and -// local credentials created by `gcloud auth application-default login`. -// When direct_serve is enabled without a private key, the GCS backend uses the -// IAM Credentials signBlob API. The service account must hold -// roles/iam.serviceAccountTokenCreator on itself. -// // Database Configuration: // // The proxy supports two database backends: @@ -60,34 +48,19 @@ package config import ( - _ "embed" "encoding/base64" "encoding/json" "fmt" - "math" - "net" "net/url" "os" - "path" "path/filepath" - "regexp" - "slices" - "sort" "strconv" "strings" "time" - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/proxy/internal/denylist" - "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/purl" "gopkg.in/yaml.v3" ) -// DefaultSwiftUpstream is the Swift Package Registry used when none is configured. -const DefaultSwiftUpstream = "https://tuist.dev/api/registry/swift" - // Config holds all configuration for the proxy server. type Config struct { // Listen is the address to listen on (e.g., ":8080", "127.0.0.1:8080"). @@ -107,31 +80,6 @@ type Config struct { // Example: "https://proxy.example.com/ui" UIBaseURL string `json:"ui_base_url" yaml:"ui_base_url"` - // TrustForwardedFor attributes requests to the leftmost X-Forwarded-For - // entry instead of the TCP peer address, in the structured log, the access - // log and the request-source table on /ui/analytics. - // - // Enable this only when the proxy sits behind a load balancer or ingress - // that sets the header, because any client can send it: behind one it is - // the only way to see past the hop, in front of one it lets a caller choose - // what address it is attributed to. - TrustForwardedFor bool `json:"trust_forwarded_for" yaml:"trust_forwarded_for"` - - // UIRequestSources shows the request-source table on /ui/analytics, which - // reports caller addresses, the tool each ran and how much each pulled. - // - // Off by default. The proxy has no authentication of its own, so until this - // is enabled /ui exposes what is cached rather than who called. - UIRequestSources bool `json:"ui_request_sources" yaml:"ui_request_sources"` - - // UIFailureDetails lists the latest failures behind each failure counter on - // /ui/analytics: the package, its upstream URL or storage path, and the - // error. URLs are shown without credentials or query strings. - // - // Off by default, for the same reason as UIRequestSources: package names and - // upstream hosts say what the proxy's users depend on. - UIFailureDetails bool `json:"ui_failure_details" yaml:"ui_failure_details"` - // Storage configures artifact storage. Storage StorageConfig `json:"storage" yaml:"storage"` @@ -141,22 +89,12 @@ type Config struct { // Log configures logging. Log LogConfig `json:"log" yaml:"log"` - // AccessLog configures the JSONL activity log. - AccessLog AccessLogConfig `json:"access_log" yaml:"access_log"` - // Upstream configures upstream registry URLs (optional overrides). Upstream UpstreamConfig `json:"upstream" yaml:"upstream"` // Cooldown configures version age filtering to mitigate supply chain attacks. Cooldown CooldownConfig `json:"cooldown" yaml:"cooldown"` - // Denylist blocks explicitly configured package versions. - Denylist DenylistConfig `json:"denylist" yaml:"denylist"` - - // Scanning configures pre-cache artifact scanning (trivy, ClamAV, Wiz, - // or a custom service) to mitigate supply chain attacks. - Scanning ScanningConfig `json:"scanning" yaml:"scanning"` - // CacheMetadata enables caching of upstream metadata responses for offline fallback. // When enabled, metadata is stored in the database and storage backend. // The mirror command always enables this regardless of this setting. @@ -172,76 +110,26 @@ type Config struct { // size return ErrMetadataTooLarge. Default: "100MB". MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"` - // MetadataStreamMaxSize is the largest metadata document the proxy - // streams from upstream into storage instead of buffering it, which - // metadata_max_size doesn't cover (e.g. "1GB"). Default: "1GB". - MetadataStreamMaxSize string `json:"metadata_stream_max_size" yaml:"metadata_stream_max_size"` - - // ArtifactMaxSize is the largest upstream artifact the proxy will fetch - // (e.g. "2GB"). Larger artifacts are refused and not cached. Default: "" - // (unlimited). "0" is also unlimited. - ArtifactMaxSize string `json:"artifact_max_size" yaml:"artifact_max_size"` - - // MetadataRewriteCacheSize is how much rewritten npm and Composer - // metadata to keep in memory, so a document is rewritten once rather than - // on every request (e.g. "256MB", "1GB"). Default: "256MB". Set to "0" to - // rewrite on every request. - MetadataRewriteCacheSize string `json:"metadata_rewrite_cache_size" yaml:"metadata_rewrite_cache_size"` - // HTTPTimeout is the timeout for individual upstream HTTP requests made // by protocol handlers (metadata fetches, pass-through file requests). // Uses Go duration syntax (e.g. "30s", "2m"). Default: "30s". - // Set to "0" to disable the timeout entirely. Independently of this - // setting, the shared transport gives up on an upstream that has not sent - // response headers within 60 seconds. + // Set to "0" to disable the timeout entirely. HTTPTimeout string `json:"http_timeout" yaml:"http_timeout"` - // WriteTimeout bounds response writes, including time spent downloading and - // scanning before the response starts. Default: "5m"; "0" disables it. - WriteTimeout string `json:"write_timeout" yaml:"write_timeout"` - - // ArtifactTimeout bounds each upstream artifact download while serving - // requests, excluding scanning. Default: "5m"; "0" disables it. - // The mirror command keeps its unlimited artifact download timeout. - ArtifactTimeout string `json:"artifact_timeout" yaml:"artifact_timeout"` - // MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP. // Disabled by default to prevent unauthenticated users from triggering downloads. MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"` - // Mirror configures scheduled mirroring. - Mirror MirrorConfig `json:"mirror" yaml:"mirror"` - - // CacheDelete enables DELETE /api/cache for removing cached artifacts, and - // the delete buttons on the UI's version page. It has no authentication of - // its own, so restrict /api/cache and /ui at the ingress when enabling it. - CacheDelete bool `json:"cache_delete" yaml:"cache_delete"` - - // Hosted configures pushing packages to the proxy (/api/hosted). - Hosted HostedConfig `json:"hosted" yaml:"hosted"` - - // Projects configures the project inventory (/api/projects). - Projects ProjectsConfig `json:"projects" yaml:"projects"` - // Gradle configures Gradle HttpBuildCache behavior. Gradle GradleConfig `json:"gradle" yaml:"gradle"` // Health configures the /health endpoint behavior. Health HealthConfig `json:"health" yaml:"health"` - - // Git configures the Git repository cache served at /git/{name}/. - Git GitConfig `json:"git" yaml:"git"` - - // envWarnings collects environment values LoadFromEnv ignored. - envWarnings []string } // CooldownConfig configures version cooldown periods. // Versions published more recently than the cooldown are hidden from metadata responses. type CooldownConfig struct { - // Profiles are independent policies exposed at /cooldown/{name}/. - Profiles map[string]CooldownConfig `json:"profiles,omitempty" yaml:"profiles,omitempty"` - // Default is the global default cooldown (e.g., "3d", "48h", "0" to disable). Default string `json:"default" yaml:"default"` @@ -249,242 +137,7 @@ type CooldownConfig struct { Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"` // Packages overrides the cooldown for specific packages (keyed by PURL). - // Valid PURL keys are normalized to canonical form before use. Packages map[string]string `json:"packages" yaml:"packages"` - - // PackagePatterns overrides the cooldown for packages whose PURLs match a glob. - // Exact package overrides take precedence over matching patterns. - PackagePatterns map[string]string `json:"package_patterns" yaml:"package_patterns"` -} - -// NormalizedPackages returns a copy of the package overrides with valid PURL -// keys in canonical form. An explicitly canonical key wins over an equivalent -// noncanonical key, and invalid keys are preserved unchanged. -func (c *CooldownConfig) NormalizedPackages() map[string]string { - if c == nil || c.Packages == nil { - return nil - } - - keys := make([]string, 0, len(c.Packages)) - for key := range c.Packages { - keys = append(keys, key) - } - sort.Strings(keys) - - normalized := make(map[string]string, len(c.Packages)) - for _, key := range keys { - canonical := key - if parsed, err := purl.Parse(key); err == nil { - canonical = packageurl.PolicyPackageKey(parsed.String()) - } - if _, exists := normalized[canonical]; exists && key != canonical { - continue - } - normalized[canonical] = c.Packages[key] - } - return normalized -} - -// Shortest returns the shortest cooldown configured anywhere, by default, -// ecosystem, package or package pattern, or 0 when none is. Unparsable -// durations are left out; Validate reports them. -func (c *CooldownConfig) Shortest() time.Duration { - if c == nil { - return 0 - } - var shortest time.Duration - consider := func(s string) { - if d, err := cooldown.ParseDuration(s); err == nil && d > 0 && (shortest == 0 || d < shortest) { - shortest = d - } - } - consider(c.Default) - for _, m := range []map[string]string{c.Ecosystems, c.Packages, c.PackagePatterns} { - for _, v := range m { - consider(v) - } - } - return shortest -} - -// ScanningConfig configures pre-cache artifact scanning (e.g. trivy, -// ClamAV, Wiz, or a custom service) to mitigate supply chain attacks. -// Unlike Cooldown, which only looks at a version's publish timestamp, -// scanning inspects the actual artifact bytes before they become -// servable from cache. -type ScanningConfig struct { - // Enabled turns on the scan gate. When false (default), artifacts are - // cached exactly as if scanning didn't exist. - Enabled bool `json:"enabled" yaml:"enabled"` - - // FailOpen treats scanner errors and timeouts as an allow verdict - // instead of a block. Default is fail-closed, since the default - // posture for a security gate should block on infrastructure failure. - FailOpen bool `json:"fail_open" yaml:"fail_open"` - - // Timeout bounds each scan call. Uses Go duration syntax (e.g. "30s"). - // Default: "30s". - Timeout string `json:"timeout" yaml:"timeout"` - - // SigningKey authenticates pull requests to the internal scan-fetch - // route used by every storage backend. Required whenever Enabled is - // true. Supports ${VAR_NAME} expansion like AuthConfig fields. - SigningKey string `json:"signing_key" yaml:"signing_key"` - - // FetchBaseURL is the address scanners use to reach this proxy to pull - // staged artifacts. Defaults to BaseURL. Set this separately when - // scanners reach the proxy over an internal address different from the - // public-facing BaseURL (mirrors DirectServeBaseURL/UIBaseURL). - FetchBaseURL string `json:"fetch_base_url" yaml:"fetch_base_url"` - - // Scanners is the list of external scanning services to call. - Scanners []ScannerConfig `json:"scanners" yaml:"scanners"` -} - -// ScannerConfig configures a single external scanning service. -type ScannerConfig struct { - // Name identifies this scanner in logs and metrics. - Name string `json:"name" yaml:"name"` - - // URL is the endpoint the proxy POSTs scan notifications to. - URL string `json:"url" yaml:"url"` - - // Mode is "block" (default) or "monitor". A "block" scanner's verdict - // can prevent caching; a "monitor" scanner's findings are logged but - // never gate caching. - Mode string `json:"mode" yaml:"mode"` - - // Ecosystems restricts this scanner to specific ecosystems (e.g. - // "npm", "pypi"). Empty means all ecosystems. - Ecosystems []string `json:"ecosystems" yaml:"ecosystems"` - - // Headers are additional HTTP headers sent with every scan request - // (e.g. for authenticating to the scanner service). Values support - // ${VAR_NAME} expansion like AuthConfig fields. - Headers map[string]string `json:"headers" yaml:"headers"` -} - -// SigningKeyExpanded returns SigningKey with ${VAR_NAME} references expanded. -func (s *ScanningConfig) SigningKeyExpanded() string { - return expandEnv(s.SigningKey) -} - -// HeadersExpanded returns Headers with ${VAR_NAME} references expanded in -// each value. -func (s *ScannerConfig) HeadersExpanded() map[string]string { - if len(s.Headers) == 0 { - return nil - } - expanded := make(map[string]string, len(s.Headers)) - for k, v := range s.Headers { - expanded[k] = expandEnv(v) - } - return expanded -} - -// Validate checks the scanning configuration for errors, applying the -// default timeout if unset. -func (s *ScanningConfig) Validate() error { - if !s.Enabled { - return nil - } - - if s.SigningKeyExpanded() == "" { - return fmt.Errorf("scanning.signing_key is required when scanning.enabled is true") - } - - if len(s.Scanners) == 0 { - return fmt.Errorf("scanning.scanners must not be empty when scanning.enabled is true") - } - - if s.FetchBaseURL != "" { - if err := validateAbsoluteURL("scanning.fetch_base_url", s.FetchBaseURL); err != nil { - return err - } - } - - if s.Timeout == "" { - s.Timeout = defaultScanningTimeoutStr - } - if d, err := time.ParseDuration(s.Timeout); err != nil { - return fmt.Errorf("invalid scanning.timeout %q: %w", s.Timeout, err) - } else if d <= 0 { - return fmt.Errorf("invalid scanning.timeout %q: must be > 0", s.Timeout) - } - - for i := range s.Scanners { - if err := s.Scanners[i].Validate(); err != nil { - return fmt.Errorf("scanning.scanners[%d]: %w", i, err) - } - } - - return nil -} - -// Validate checks a single scanner's configuration, applying the default -// mode ("block") if unset. -func (s *ScannerConfig) Validate() error { - if s.Name == "" { - return fmt.Errorf("name is required") - } - if err := validateAbsoluteURL("url", s.URL); err != nil { - return err - } - if s.Mode == "" { - s.Mode = "block" - } - switch s.Mode { - case "block", "monitor": - default: - return fmt.Errorf("invalid mode %q (must be block or monitor)", s.Mode) - } - for i, eco := range s.Ecosystems { - key, err := ScannerEcosystemKey(eco) - if err != nil { - return err - } - s.Ecosystems[i] = key - } - return nil -} - -// scannerEcosystems are the ecosystem keys handlers pass to scanners: the -// retention keys plus the handlers retention does not cover, and git for -// Git LFS objects. -var scannerEcosystems = func() []string { - keys := append(slices.Clone(retention.KnownKeys), "arch", "chef", "git") - slices.Sort(keys) - return keys -}() - -// scannerEcosystemAliases maps common ecosystem names to handler keys. -var scannerEcosystemAliases = map[string]string{ - "apk": "alpine", - "debian": "deb", - "docker": "oci", - "go": "golang", - "packagist": "composer", - "rubygems": "gem", -} - -// NormalizeScannerEcosystem returns the handler key for a -// scanning.scanners[].ecosystems entry, resolving case and common aliases. -func NormalizeScannerEcosystem(eco string) string { - eco = strings.ToLower(strings.TrimSpace(eco)) - if key, ok := scannerEcosystemAliases[eco]; ok { - return key - } - return eco -} - -// ScannerEcosystemKey is NormalizeScannerEcosystem that also rejects -// names no handler uses. -func ScannerEcosystemKey(eco string) (string, error) { - key := NormalizeScannerEcosystem(eco) - if !slices.Contains(scannerEcosystems, key) { - return "", fmt.Errorf("unknown ecosystem %q (valid: %s)", eco, strings.Join(scannerEcosystems, ", ")) - } - return key, nil } // StorageConfig configures artifact storage. @@ -494,8 +147,6 @@ type StorageConfig struct { // - file:///path/to/dir - Local filesystem (default) // - s3://bucket-name - Amazon S3 // - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO) - // - gs://bucket-name - Google Cloud Storage (Workload Identity supported) - // - azblob://container-name - Azure Blob Storage // If empty, defaults to file:// with the Path value. URL string `json:"url" yaml:"url"` @@ -512,7 +163,7 @@ type StorageConfig struct { // DirectServe enables redirecting cached artifact downloads to presigned // storage URLs (HTTP 302) instead of streaming bytes through the proxy. - // Only effective for backends that support URL signing (S3, GCS, Azure). + // Only effective for backends that support URL signing (S3, Azure). DirectServe bool `json:"direct_serve" yaml:"direct_serve"` // DirectServeTTL is how long presigned URLs remain valid. @@ -524,41 +175,6 @@ type StorageConfig struct { // storage at an internal address (e.g. 127.0.0.1 or a Docker hostname) // but clients must use a public one. DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"` - - // CacheArtifacts stores fetched artifacts so later downloads are served - // from storage. When false, every download streams from upstream and - // nothing is stored; metadata is still cached, and cooldown and the - // denylist still apply. Useful when another caching layer sits in front - // of the proxy. False is incompatible with scanning, direct_serve and - // mirror_api, which all depend on stored artifacts. Default: true. - CacheArtifacts bool `json:"cache_artifacts" yaml:"cache_artifacts"` - - // Retention evicts cached artifacts that have not been accessed for a - // configured time, per ecosystem and per package. - Retention RetentionConfig `json:"retention" yaml:"retention"` -} - -// RetentionConfig configures age-based eviction of cached artifacts. An -// artifact's age is the time since it was last served, or since it was -// fetched when it has not been served since. Durations accept a "d" suffix -// for days; "0" or empty means never evict by age. -type RetentionConfig struct { - // Default applies to every ecosystem that supports retention and has no - // rule of its own, except generic, which requires an explicit rule. - Default string `json:"default" yaml:"default"` - - // Ecosystems overrides the default per ecosystem key (e.g. "npm", "oci"). - // Configuring an ecosystem whose retention is not supported yet is an - // error. - Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"` - - // Packages overrides the ecosystem rule for single packages, keyed by - // package PURL. - Packages map[string]string `json:"packages" yaml:"packages"` - - // SweepInterval is how often expired artifacts are looked for, at least - // one minute. Default: "10m". - SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"` } // GradleConfig configures Gradle-specific features. @@ -570,7 +186,6 @@ type GradleConfig struct { // GradleBuildCacheConfig configures Gradle HttpBuildCache safeguards. type GradleBuildCacheConfig struct { // ReadOnly disables PUT uploads and keeps cache reads (GET/HEAD) enabled. - // Default: true; set false on the instance that CI uploads to. ReadOnly bool `json:"read_only" yaml:"read_only"` // MaxUploadSize caps a single PUT body size (e.g., "100MB"). Must be > 0. @@ -608,12 +223,6 @@ type DatabaseConfig struct { // URL is the PostgreSQL connection string. URL string `json:"url" yaml:"url"` - - // HitFlushInterval is how often cache hit counts and last-access times - // are written, batched in one transaction. Uses Go duration syntax - // (e.g. "1s", "10s"). Default: "1s". Set to "0" to write each hit as it - // happens. - HitFlushInterval string `json:"hit_flush_interval" yaml:"hit_flush_interval"` } // String returns a human-readable description of the configured database @@ -640,44 +249,21 @@ type LogConfig struct { Format string `json:"format" yaml:"format"` } -// AccessLogConfig configures the JSONL activity log. -type AccessLogConfig struct { - // Path is the file to append activity records to. Empty disables the access log. - Path string `json:"path" yaml:"path"` -} - -// UpstreamConfig configures upstream URLs for built-in routes and authentication. +// UpstreamConfig configures upstream registry URLs and authentication. // Leave empty to use defaults. type UpstreamConfig struct { - // AllowPrivateHosts permits listed upstream hosts to resolve to private addresses. - AllowPrivateHosts []string `json:"allow_private_hosts" yaml:"allow_private_hosts"` - - // AllowLoopback permits upstream requests and redirects to loopback addresses. - AllowLoopback bool `json:"allow_loopback" yaml:"allow_loopback"` - // NPM is the upstream npm registry URL. // Default: https://registry.npmjs.org NPM string `json:"npm" yaml:"npm"` - // NPMFullMetadata always requests the full packument (application/json) - // from the npm upstream, so served metadata carries the "time" map even - // when cooldown is disabled. Clients that gate on publish age (for - // example Yarn's npmMinimalAgeGate) need this. - // Default: false (the abbreviated format is preferred). - NPMFullMetadata bool `json:"npm_full_metadata" yaml:"npm_full_metadata"` + // Maven is the upstream Maven repository URL. + // Default: https://repo1.maven.org/maven2 + Maven string `json:"maven" yaml:"maven"` - // NPMRoutes maps package name patterns to npm registry URLs. A package - // whose name matches a pattern is fetched only from that registry, never - // from upstream.npm, so a public package cannot stand in for it. Patterns - // use path.Match syntax, e.g. "@example/*"; the longest matching pattern - // wins. - NPMRoutes map[string]string `json:"npm_routes" yaml:"npm_routes"` - - // NPMChangeFeed follows the public npm registry's change feed and - // revalidates cached packuments of packages listed in it on their next - // request, rather than serving them until metadata_ttl runs out. - // Requires cache_metadata. Default: false. - NPMChangeFeed bool `json:"npm_change_feed" yaml:"npm_change_feed"` + // GradlePluginPortal is the upstream Gradle Plugin Portal Maven URL. + // Used to resolve Gradle plugin marker artifacts. + // Default: https://plugins.gradle.org/m2 + GradlePluginPortal string `json:"gradle_plugin_portal" yaml:"gradle_plugin_portal"` // Cargo is the upstream cargo index URL. // Default: https://index.crates.io @@ -687,249 +273,24 @@ type UpstreamConfig struct { // Default: https://static.crates.io/crates CargoDownload string `json:"cargo_download" yaml:"cargo_download"` - // Gem is the upstream RubyGems registry URL. - // Default: https://rubygems.org - Gem string `json:"gem" yaml:"gem"` - - // Go is the upstream Go module proxy URL. - // Default: https://proxy.golang.org - Go string `json:"go" yaml:"go"` - - // GoSumDBs maps Go checksum database names to their base URLs, served at - // /go/sumdb/{name}/ so the go command reaches the database through - // GOPROXY. Unknown names get 404, which makes the go command contact the - // database directly. - // Default when empty: {"sum.golang.org": "https://sum.golang.org"}. - GoSumDBs map[string]string `json:"go_sumdbs" yaml:"go_sumdbs"` - - // GoSumDBsOffline keeps answering for a checksum database from its - // cached signed tree, however old, when the database cannot be reached. - // Set this when clients have no route to the database of their own; - // otherwise the proxy stops answering once the cached tree is older than - // metadata_ttl, and the go command contacts the database directly. - // Requires cache_metadata. Default: false. - GoSumDBsOffline bool `json:"go_sumdbs_offline" yaml:"go_sumdbs_offline"` - - // GoVulnDB is the Go vulnerability database served at /go/vulndb, for - // govulncheck -db pointing at the proxy. - // Default: https://vuln.go.dev - GoVulnDB string `json:"go_vulndb" yaml:"go_vulndb"` - - // Hex is the upstream Hex repository URL. - // Default: https://repo.hex.pm - Hex string `json:"hex" yaml:"hex"` - - // HexAPI is the upstream Hex API URL used for package timestamps. - // Default: https://hex.pm - HexAPI string `json:"hex_api" yaml:"hex_api"` - - // Pub is the upstream pub registry URL. - // Default: https://pub.dev - Pub string `json:"pub" yaml:"pub"` - - // PyPI is the upstream PyPI index and API URL. - // Default: https://pypi.org - PyPI string `json:"pypi" yaml:"pypi"` - - // PyPIDownload is the upstream PyPI package download URL. - // Default: https://files.pythonhosted.org - PyPIDownload string `json:"pypi_download" yaml:"pypi_download"` - - // Maven is the upstream Maven repository URL. - // Default: https://repo1.maven.org/maven2 - Maven string `json:"maven" yaml:"maven"` - - // MavenRepositories maps repository names to additional Maven repository - // base URLs, served at /maven/{name}/. upstream.maven stays the default - // at /maven/. A name must not be the first segment of a group ID, such - // as "org" or "tools", since it would hide those groups in the default - // repository; see validateMavenRepositoryName. - // Example: {"grails": "https://repo.grails.org/grails/restricted"}. - MavenRepositories map[string]string `json:"maven_repositories" yaml:"maven_repositories"` - - // GradlePluginPortal is the upstream Gradle Plugin Portal Maven URL. - // Used to resolve Gradle plugin marker artifacts. - // Default: https://plugins.gradle.org/m2 - GradlePluginPortal string `json:"gradle_plugin_portal" yaml:"gradle_plugin_portal"` - - // NuGet is the upstream NuGet API URL. - // Default: https://api.nuget.org - NuGet string `json:"nuget" yaml:"nuget"` - - // NuGetSearch is the upstream NuGet search API URL. - // Default: https://azuresearch-usnc.nuget.org - NuGetSearch string `json:"nuget_search" yaml:"nuget_search"` - - // Composer is the upstream Packagist API URL. - // Default: https://packagist.org - Composer string `json:"composer" yaml:"composer"` - - // ComposerRepository is the upstream Packagist repository URL. - // Default: https://repo.packagist.org - ComposerRepository string `json:"composer_repository" yaml:"composer_repository"` - - // ComposerRoutes maps package name patterns to Composer repository URLs - // (the base that serves /p2/{vendor}/{package}.json). A package whose - // name matches a pattern is fetched only from that repository, never - // from upstream.composer_repository, so a public package cannot stand in - // for it. Patterns use path.Match syntax, e.g. "example/*"; the longest - // matching pattern wins. - ComposerRoutes map[string]string `json:"composer_routes" yaml:"composer_routes"` - - // ComposerChangeFeed follows the metadata change feed of - // upstream.composer and revalidates cached metadata of packages listed - // in it on their next request, rather than serving it until - // metadata_ttl runs out. Requires cache_metadata. Default: false. - ComposerChangeFeed bool `json:"composer_change_feed" yaml:"composer_change_feed"` - - // Conan is the upstream Conan registry URL. - // Default: https://center.conan.io - Conan string `json:"conan" yaml:"conan"` - - // Chef is the upstream Chef Supermarket URL, served at /chef/. Point it - // at a private Supermarket to proxy that instead of the public one. - // Default: https://supermarket.chef.io - Chef string `json:"chef" yaml:"chef"` - - // Conda is the upstream Conda channel base URL. - // Default: https://conda.anaconda.org - Conda string `json:"conda" yaml:"conda"` - - // CRAN is the upstream CRAN mirror URL. - // Default: https://cloud.r-project.org - CRAN string `json:"cran" yaml:"cran"` - - // Julia is the upstream Julia package server URL. - // Default: https://pkg.julialang.org - Julia string `json:"julia" yaml:"julia"` - - // OCIDefault is the default upstream OCI registry URL. - // Default: https://registry-1.docker.io - OCIDefault string `json:"oci_default" yaml:"oci_default"` - - // Swift is the upstream Swift Package Registry URL. - // Default: https://tuist.dev/api/registry/swift - Swift string `json:"swift" yaml:"swift"` - - // Debian is the upstream APT repository base URL, served at /debian/. - // Example: http://archive.ubuntu.com/ubuntu would get Ubuntu. - // Default: http://deb.debian.org/debian - Debian string `json:"debian" yaml:"debian"` - - // DebianRepositories maps repository names to additional APT repository - // base URLs, served at /debian/{name}/. - // Example: {"security": "https://security.debian.org/debian-security"}. - DebianRepositories map[string]string `json:"debian_repositories" yaml:"debian_repositories"` - - // RPM is the upstream RPM repository base URL. - // Default: https://dl.fedoraproject.org/pub/fedora/linux - RPM string `json:"rpm" yaml:"rpm"` - - // HomebrewAPI is the upstream Homebrew JSON API URL. - // Default: https://formulae.brew.sh/api - HomebrewAPI string `json:"homebrew_api" yaml:"homebrew_api"` - - // HomebrewArtifact is the upstream registry URL for Homebrew artifacts. - // Default: https://ghcr.io - HomebrewArtifact string `json:"homebrew_artifact" yaml:"homebrew_artifact"` - - // Helm maps repository names to HTTP Helm chart repository URLs. - // Requests use /helm/{name}/index.yaml and chart URLs in the index are - // rewritten to the same named proxy endpoint. - Helm map[string]string `json:"helm" yaml:"helm"` - - // APK maps repository names to Alpine APK repository base URLs, served - // at /apk/{name}/. The remaining request path mirrors the upstream - // layout, e.g. /apk/alpine/v3.22/main/x86_64/APKINDEX.tar.gz. - // Default when empty: {"alpine": "https://dl-cdn.alpinelinux.org/alpine"}. - APK map[string]string `json:"apk" yaml:"apk"` - - // APKSnapshots keeps earlier versions of Alpine APK indexes for this - // long after upstream replaced them, and serves them at - // /apk/{name}@{YYYY-MM-DD}/. Accepts a "d" suffix, e.g. "90d". - // Empty or "0" disables snapshots. - APKSnapshots string `json:"apk_snapshots" yaml:"apk_snapshots"` - - // Arch maps repository names to Arch Linux pacman mirror base URLs, - // served at /arch/{name}/. The remaining request path mirrors the - // upstream layout, e.g. /arch/archlinux/core/os/x86_64/core.db. - // Default when empty: {"archlinux": "https://geo.mirror.pkgbuild.com"}. - Arch map[string]string `json:"arch" yaml:"arch"` - - // FreeBSD maps repository names to FreeBSD pkg mirror base URLs, served - // at /freebsd/{name}/. The remaining request path mirrors the upstream - // layout, e.g. /freebsd/freebsd/FreeBSD:14:amd64/latest/packagesite.pkg. - // Default when empty: {"freebsd": "https://pkg.freebsd.org"}. - FreeBSD map[string]string `json:"freebsd" yaml:"freebsd"` - - // OCI maps names to OCI registry URLs. Requests to a named registry use - // the repository prefix upstream/{name}/, for example - // oci://proxy.example.com/upstream/ghcr/owner/chart. - OCI map[string]string `json:"oci" yaml:"oci"` - - // OCIMirrors lists, per upstream.oci name, the registry hosts that - // upstream mirrors, for example {"ghcr": ["ghcr.io"]} for an Artifactory - // remote of ghcr.io. containerd requests whose ns query parameter names - // one of these hosts are served by that upstream. - OCIMirrors map[string][]string `json:"oci_mirrors" yaml:"oci_mirrors"` - - // Generic maps names to plain HTTP upstream base URLs, served at - // /generic/{name}/. The remaining request path and query string are - // appended to the upstream URL. GitHub release asset paths - // ({owner}/{repo}/releases/download/{tag}/{asset}) and paths matching - // GenericArtifacts are cached in the artifact cache; everything else - // goes through the metadata cache. - // Example: {"github": "https://github.com", "github-api": "https://api.github.com"}. - Generic map[string]string `json:"generic" yaml:"generic"` - - // GenericArtifacts maps a Generic upstream name to regular expressions for - // its version-pinned download paths. A request path after - // /generic/{name}/ that one of them matches in full is cached in the - // artifact cache like a GitHub release asset. Each pattern needs `version` - // and `file` named groups; an optional `name` group names the package, - // which otherwise is the upstream name. - // Example: {"apache": ["dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)"]}. - GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"` - - // GenericPaths maps a Generic upstream name to the paths it may serve, - // as patterns for the request path after /generic/{name}/ with the same - // syntax as GitUpstream.Repos: path.Match, where a pattern ending in - // "/**" also matches every path nested below its prefix. Other paths - // answer 404. Empty serves every path. Required when upstream.auth has - // credentials for the upstream, since everyone who can reach the proxy - // can use them for any path otherwise. - // Example: {"github-api": ["repos/*/*/releases/**"]}. - GenericPaths map[string][]string `json:"generic_paths" yaml:"generic_paths"` - - // Git maps names to Git servers, served read-only at /git/{name}/{repo} - // from a local cache of bare repositories. See GitUpstream. - // Example: {"github": {"url": "https://github.com"}}. - Git map[string]GitUpstream `json:"git" yaml:"git"` - // Auth configures authentication for upstream registries. - // Keys are absolute URL scopes matched by scheme, host, effective port, - // and path-segment prefix. + // Keys are URL prefixes that are matched against request URLs. // Example: "https://npm.pkg.github.com" matches all requests to that host. Auth map[string]AuthConfig `json:"auth" yaml:"auth"` } // AuthForURL returns the auth config that matches the given URL. -// The longest matching URL scope wins. +// Matches are based on URL prefix - the longest matching prefix wins. func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig { if u.Auth == nil { return nil } - target, err := parseAuthURL(url) - if err != nil { - return nil - } var bestMatch *AuthConfig var bestLen int for pattern, auth := range u.Auth { - configured, err := parseAuthURL(pattern) - if err == nil && authURLMatches(configured, target) && len(pattern) > bestLen { + if strings.HasPrefix(url, pattern) && len(pattern) > bestLen { a := auth // copy to avoid loop variable capture bestMatch = &a bestLen = len(pattern) @@ -939,377 +300,9 @@ func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig { return bestMatch } -// Validate checks upstream authentication URL scopes. -func (u *UpstreamConfig) Validate() error { - for pattern := range u.Auth { - if _, err := parseAuthURL(pattern); err != nil { - return fmt.Errorf("invalid upstream.auth URL %q: %w", pattern, err) - } - } - if err := validateNamedUpstreams("upstream.helm", u.Helm); err != nil { - return err - } - if err := validatePackageRoutes("upstream.composer_routes", u.ComposerRoutes); err != nil { - return err - } - if err := validatePackageRoutes("upstream.npm_routes", u.NPMRoutes); err != nil { - return err - } - if err := validateNamedUpstreams("upstream.apk", u.APK); err != nil { - return err - } - if u.APKSnapshots != "" { - if _, err := parseRetentionDuration("upstream.apk_snapshots", u.APKSnapshots); err != nil { - return err - } - } - if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil { - return err - } - if err := validateNamedUpstreams("upstream.freebsd", u.FreeBSD); err != nil { - return err - } - if err := validateNamedUpstreams("upstream.go_sumdbs", u.GoSumDBs); err != nil { - return err - } - if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil { - return err - } - if err := validateOCIMirrors(u.OCIMirrors, u.OCI); err != nil { - return err - } - if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil { - return err - } - if err := u.validateGenericArtifacts(); err != nil { - return err - } - if err := u.validateGenericPaths(); err != nil { - return err - } - if err := u.validateGitUpstreams(); err != nil { - return err - } - if err := validateNamedUpstreams("upstream.maven_repositories", u.MavenRepositories); err != nil { - return err - } - for name := range u.MavenRepositories { - if err := validateMavenRepositoryName(name); err != nil { - return err - } - } - if err := validateNamedUpstreams("upstream.debian_repositories", u.DebianRepositories); err != nil { - return err - } - for _, name := range debianReservedRepositoryNames { - if _, found := u.DebianRepositories[name]; found { - return fmt.Errorf( - "invalid upstream.debian_repositories name %q: reserved for the upstream.debian archive", - name, - ) - } - } - return nil -} - -func (u *UpstreamConfig) validateGenericArtifacts() error { - for name, patterns := range u.GenericArtifacts { - if _, found := u.Generic[name]; !found { - return fmt.Errorf("invalid upstream.generic_artifacts name %q: no upstream.generic entry", name) - } - for _, pattern := range patterns { - if _, err := CompileGenericArtifactPattern(pattern); err != nil { - return fmt.Errorf("invalid upstream.generic_artifacts.%s pattern %q: %w", name, pattern, err) - } - } - } - return nil -} - -// validateGenericPaths checks upstream.generic_paths patterns. An upstream that -// upstream.auth has credentials for must list its paths, so configuring a -// token does not let everyone who can reach the proxy send any request with -// it, mirroring the repos rule for upstream.git. -func (u *UpstreamConfig) validateGenericPaths() error { - for name, patterns := range u.GenericPaths { - if _, found := u.Generic[name]; !found { - return fmt.Errorf("invalid upstream.generic_paths name %q: no upstream.generic entry", name) - } - for _, pattern := range patterns { - if _, err := path.Match(strings.TrimSuffix(pattern, "/**"), ""); pattern == "" || err != nil { - return fmt.Errorf("invalid upstream.generic_paths.%s pattern %q", name, pattern) - } - } - } - for name, upstreamURL := range u.Generic { - if len(u.GenericPaths[name]) == 0 && u.HasAuthWithin(upstreamURL) { - return fmt.Errorf("upstream.generic.%s needs upstream.generic_paths.%s: upstream.auth has credentials for %s, "+ - "and everyone who can reach the proxy could send any request with them", name, name, upstreamURL) - } - } - return nil -} - -// GenericPathAllowed reports whether p, a request path after -// /generic/{name}/, matches one of an upstream's upstream.generic_paths -// patterns. No patterns allow every path. With patterns, a path holding a -// character the upstream could read as the end of the path or as a different -// path ('%', '?', '#' or a backslash) is refused, so the path that matched is -// the one the upstream sees. -func GenericPathAllowed(patterns []string, p string) bool { - if len(patterns) == 0 { - return true - } - if strings.ContainsAny(p, `%?#\`) { - return false - } - for _, pattern := range patterns { - if matchRepoPattern(pattern, p) { - return true - } - } - return false -} - -// validateUpstream checks the upstream block, including the one setting in -// it that depends on a top-level option. -func (c *Config) validateUpstream() error { - if err := c.Upstream.Validate(); err != nil { - return err - } - if c.Upstream.GoSumDBsOffline && !c.CacheMetadata { - return fmt.Errorf("upstream.go_sumdbs_offline requires cache_metadata") - } - if c.Upstream.NPMChangeFeed && !c.CacheMetadata { - return fmt.Errorf("upstream.npm_change_feed requires cache_metadata") - } - if c.Upstream.ComposerChangeFeed && !c.CacheMetadata { - return fmt.Errorf("upstream.composer_change_feed requires cache_metadata") - } - return nil -} - -// HostedGenericRepository is the /generic/{name}/ path that serves files -// pushed through /api/hosted/generic. While hosted packages are enabled, no -// upstream.generic entry may use it. -const HostedGenericRepository = "hosted" - -// CompileGenericArtifactPattern compiles an upstream.generic_artifacts pattern -// so that it only matches a whole path, and checks it has the `version` and -// `file` groups. -func CompileGenericArtifactPattern(pattern string) (*regexp.Regexp, error) { - re, err := regexp.Compile(`^(?:` + pattern + `)$`) - if err != nil { - return nil, err - } - for _, group := range []string{"version", "file"} { - if re.SubexpIndex(group) < 0 { - return nil, fmt.Errorf("missing (?P<%s>...) group", group) - } - } - return re, nil -} - -// mavenGroupRootList lists the first path segments of group IDs on Maven -// Central and Google's Maven repository, one per line. -// -//go:embed maven_group_roots.txt -var mavenGroupRootList string - -// mavenGroupRoots are the first segments of group IDs a repository name -// must not take: those in mavenGroupRootList, and generic top-level domains -// group IDs are commonly reversed from. -var mavenGroupRoots = func() map[string]bool { - roots := map[string]bool{} - for _, tld := range []string{"app", "biz", "cloud", "com", "dev", "edu", "gov", "info", "int", - "io", "mil", "name", "net", "org", "pro", "tech", "xyz"} { - roots[tld] = true - } - for line := range strings.Lines(mavenGroupRootList) { - if line = strings.TrimSpace(line); line != "" && !strings.HasPrefix(line, "#") { - roots[line] = true - } - } - return roots -}() - -// mavenRepositoryNamePattern is what a repository name may look like: the -// characters of a group ID segment, which clients send unescaped. -var mavenRepositoryNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`) - -// countryCodeLength is the length of a country-code top-level domain. -const countryCodeLength = 2 - -// validateMavenRepositoryName refuses names that are likely the first -// segment of a group ID. /maven/{name}/ is matched before the default -// repository, so such a name would hide every group under it there: -// a repository named "tools" would take /maven/tools/jackson/... away from -// Maven Central. Names are limited to characters that reach the handler -// unescaped. Paths are case-sensitive, so names are compared as they -// are. Names of two characters or less are refused as country codes. The -// list can't be complete, since group IDs are free-form, so the -// documentation also asks for names that can't be a group root. -func validateMavenRepositoryName(name string) error { - if !mavenRepositoryNamePattern.MatchString(name) { - return fmt.Errorf("invalid upstream.maven_repositories name %q: use letters, digits, '.', '_' and '-', "+ - "starting with a letter or digit, so the name works as a URL path segment", name) - } - if len(name) <= countryCodeLength { - return fmt.Errorf("invalid upstream.maven_repositories name %q: names of two characters or less "+ - "could hide country-code group IDs in the default repository", name) - } - if mavenGroupRoots[name] { - return fmt.Errorf("invalid upstream.maven_repositories name %q: it would hide group IDs starting with %q "+ - "in the default repository; use a name such as \"%s-repo\"", name, name, name) - } - return nil -} - -// debianReservedRepositoryNames are the upstream.debian archive's own root -// paths, which a repository of the same name would shadow. -var debianReservedRepositoryNames = []string{"pool", "dists"} - -// validateOCIMirrors checks that every upstream.oci_mirrors entry belongs to -// an upstream.oci name and lists bare registry hosts, the form containerd -// sends in the ns query parameter. A host may belong to one upstream only; -// otherwise both prefixes would take it while unprefixed requests silently -// went to one of them. -func validateOCIMirrors(mirrors map[string][]string, upstreams map[string]string) error { - names := make([]string, 0, len(mirrors)) - for name := range mirrors { - names = append(names, name) - } - sort.Strings(names) - owners := make(map[string]string) - for _, name := range names { - if _, ok := upstreams[name]; !ok { - return fmt.Errorf("invalid upstream.oci_mirrors name %q: no upstream.oci entry of that name", name) - } - for _, host := range mirrors[name] { - parsed, err := url.Parse("https://" + host) - if host == "" || err != nil || parsed.Host != host { - return fmt.Errorf("invalid upstream.oci_mirrors.%s host %q: must be a registry host such as ghcr.io or registry.example:5000", name, host) - } - key := ociMirrorHostKey(host) - if owner, taken := owners[key]; taken && owner != name { - return fmt.Errorf("invalid upstream.oci_mirrors.%s host %q: already listed for %s", name, host, owner) - } - owners[key] = name - } - } - return nil -} - -// ociMirrorHostKey normalizes an upstream.oci_mirrors host the way the -// container handler builds its ns lookup keys: lowercase, IPv6 literals in -// brackets, an empty or default https port dropped. Two entries with the same -// key would select the same route. -func ociMirrorHostKey(host string) string { - name, port, err := net.SplitHostPort(host) - if err != nil { - name, port = strings.TrimSuffix(strings.TrimPrefix(host, "["), "]"), "" - } - name = strings.ToLower(name) - if strings.Contains(name, ":") { - name = "[" + name + "]" - } - if port == "" || port == "443" { - return name - } - return name + ":" + port -} - -func validateNamedUpstreams(field string, upstreams map[string]string) error { - for name, upstreamURL := range upstreams { - if name == "" || name == "." || name == ".." || strings.ContainsAny(name, `/\\`) { - return fmt.Errorf("invalid %s name %q", field, name) - } - if err := validateAbsoluteURL(field+"."+name, upstreamURL); err != nil { - return err - } - } - return nil -} - -func validatePackageRoutes(field string, routes map[string]string) error { - for pattern, upstreamURL := range routes { - if _, err := path.Match(pattern, ""); pattern == "" || err != nil { - return fmt.Errorf("invalid %s pattern %q", field, pattern) - } - if err := validateAbsoluteURL(field+"."+pattern, upstreamURL); err != nil { - return err - } - } - return nil -} - -func parseAuthURL(value string) (*url.URL, error) { - parsed, err := url.Parse(value) - if err != nil || !parsed.IsAbs() || parsed.Hostname() == "" || parsed.Opaque != "" { - return nil, fmt.Errorf("invalid authentication URL") - } - return parsed, nil -} - -func authURLMatches(configured, target *url.URL) bool { - // An upstream resolves dot segments, so /scope/../other would leave the - // configured path while still matching it by prefix. Real package URLs - // never contain them, so they get no credentials at all. - if HasDotSegment(target.Path) { - return false - } - if !strings.EqualFold(configured.Scheme, target.Scheme) || - !strings.EqualFold(configured.Hostname(), target.Hostname()) || - authURLPort(configured) != authURLPort(target) { - return false - } - if configured.RawQuery != "" && configured.RawQuery != target.RawQuery { - return false - } - - configuredPath := strings.TrimSuffix(configured.EscapedPath(), "/") - if configuredPath == "" { - return true - } - targetPath := strings.TrimSuffix(target.EscapedPath(), "/") - return targetPath == configuredPath || strings.HasPrefix(targetPath, configuredPath+"/") -} - -// HasDotSegment reports whether the URL path p, or p decoded once more, has a -// '.' or '..' segment. Backslashes count as separators, and ";params" are -// ignored, since some upstream servers (Tomcat, Jetty) treat them that way. -func HasDotSegment(p string) bool { - candidates := []string{p} - if decoded, err := url.PathUnescape(p); err == nil { - candidates = append(candidates, decoded) - } - for _, candidate := range candidates { - for segment := range strings.SplitSeq(strings.ReplaceAll(candidate, `\`, "/"), "/") { - segment, _, _ = strings.Cut(segment, ";") - if segment == "." || segment == ".." { - return true - } - } - } - return false -} - -func authURLPort(value *url.URL) string { - if port := value.Port(); port != "" { - return port - } - if strings.EqualFold(value.Scheme, "https") { - return "443" - } - if strings.EqualFold(value.Scheme, "http") { - return "80" - } - return "" -} - // AuthConfig configures authentication for an upstream registry. type AuthConfig struct { - // Type is the authentication type: "bearer", "basic", "header", or "ecr". + // Type is the authentication type: "bearer", "basic", or "header". Type string `json:"type" yaml:"type"` // Token is used for bearer authentication. @@ -1329,11 +322,6 @@ type AuthConfig struct { // HeaderValue is the custom header value (for type "header"). // Can reference environment variables with ${VAR_NAME} syntax. HeaderValue string `json:"header_value" yaml:"header_value"` - - // Region is the AWS region for ECR authentication (for type "ecr"). - // If empty, the region is inferred from private ECR registry URLs before - // falling back to the AWS SDK default region chain. - Region string `json:"region" yaml:"region"` } // Default returns a Config with sensible defaults. @@ -1342,12 +330,8 @@ func Default() *Config { Listen: ":8080", BaseURL: "http://localhost:8080", Storage: StorageConfig{ - Path: "./cache/artifacts", - MaxSize: "", - CacheArtifacts: true, - Retention: RetentionConfig{ - SweepInterval: defaultRetentionSweepIntervalStr, - }, + Path: "./cache/artifacts", + MaxSize: "", }, Database: DatabaseConfig{ Driver: "sqlite", @@ -1359,37 +343,14 @@ func Default() *Config { }, Upstream: UpstreamConfig{ NPM: "https://registry.npmjs.org", - Cargo: "https://index.crates.io", - CargoDownload: "https://static.crates.io/crates", - Gem: "https://rubygems.org", - Go: "https://proxy.golang.org", - GoVulnDB: "https://vuln.go.dev", - Hex: "https://repo.hex.pm", - HexAPI: "https://hex.pm", - Pub: "https://pub.dev", - PyPI: "https://pypi.org", - PyPIDownload: "https://files.pythonhosted.org", Maven: "https://repo1.maven.org/maven2", GradlePluginPortal: "https://plugins.gradle.org/m2", - NuGet: "https://api.nuget.org", - NuGetSearch: "https://azuresearch-usnc.nuget.org", - Composer: "https://packagist.org", - ComposerRepository: "https://repo.packagist.org", - Conan: "https://center.conan.io", - Chef: "https://supermarket.chef.io", - Conda: "https://conda.anaconda.org", - CRAN: "https://cloud.r-project.org", - Julia: "https://pkg.julialang.org", - Swift: DefaultSwiftUpstream, - OCIDefault: "https://registry-1.docker.io", - Debian: "http://deb.debian.org/debian", - RPM: "https://dl.fedoraproject.org/pub/fedora/linux", - HomebrewAPI: "https://formulae.brew.sh/api", - HomebrewArtifact: "https://ghcr.io", + Cargo: "https://index.crates.io", + CargoDownload: "https://static.crates.io/crates", }, Gradle: GradleConfig{ BuildCache: GradleBuildCacheConfig{ - ReadOnly: true, + ReadOnly: false, MaxUploadSize: defaultGradleMaxUploadSizeStr, MaxAge: "168h", MaxSize: "", @@ -1430,81 +391,6 @@ func Load(path string) (*Config, error) { return cfg, nil } -// setEnvString sets *dst from the named environment variable, leaving it -// untouched if the variable is unset or empty. -func setEnvString(dst *string, key string) { - if v := os.Getenv(key); v != "" { - *dst = v - } -} - -// parseEnvBool accepts what strconv.ParseBool does plus yes/no/on/off, -// ignoring case and surrounding space. -func parseEnvBool(v string) (bool, bool) { - v = strings.ToLower(strings.TrimSpace(v)) - switch v { - case "yes", "on": - return true, true - case "no", "off": - return false, true - } - b, err := strconv.ParseBool(v) - return b, err == nil -} - -// setEnvBool is setEnvString for boolean fields, parsed via parseEnvBool. -// Any other value leaves *dst unchanged and is reported via EnvWarnings. -func (c *Config) setEnvBool(dst *bool, key string) { - v := os.Getenv(key) - if v == "" { - return - } - b, ok := parseEnvBool(v) - if !ok { - c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping %t", key, v, *dst)) - return - } - *dst = b -} - -// setGradleReadOnlyFromEnv parses strictly: an unrecognised value keeps -// uploads off instead of opening them, and is reported via EnvWarnings. -func (c *Config) setGradleReadOnlyFromEnv() { - const key = "PROXY_GRADLE_BUILD_CACHE_READ_ONLY" - v := os.Getenv(key) - if v == "" { - return - } - readOnly, ok := parseEnvBool(v) - if !ok { - c.Gradle.BuildCache.ReadOnly = true - c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping the Gradle build cache read-only", key, v)) - return - } - c.Gradle.BuildCache.ReadOnly = readOnly -} - -// EnvWarnings returns the environment values LoadFromEnv ignored. -func (c *Config) EnvWarnings() []string { - return c.envWarnings -} - -func setEnvStringSlice(dst *[]string, key string) { - value := os.Getenv(key) - if value == "" { - return - } - var items []string - for item := range strings.SplitSeq(value, ",") { - if item = strings.TrimSpace(item); item != "" { - items = append(items, item) - } - } - if len(items) > 0 { - *dst = items - } -} - // LoadFromEnv applies environment variable overrides to a Config. // Environment variables use the PROXY_ prefix: // - PROXY_LISTEN @@ -1515,105 +401,92 @@ func setEnvStringSlice(dst *[]string, key string) { // - PROXY_DATABASE_PATH // - PROXY_LOG_LEVEL // - PROXY_LOG_FORMAT -// - PROXY_ACCESS_LOG_PATH -// - PROXY_TRUST_FORWARDED_FOR -// - PROXY_UI_REQUEST_SOURCES -// - PROXY_UPSTREAM_SWIFT // - PROXY_HEALTH_STORAGE_PROBE_INTERVAL func (c *Config) LoadFromEnv() { - setEnvString(&c.Listen, "PROXY_LISTEN") - setEnvString(&c.BaseURL, "PROXY_BASE_URL") - setEnvString(&c.UIBaseURL, "PROXY_UI_URL") - c.setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR") - c.setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES") - c.setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS") - setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL") - setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH") - setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE") - c.setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") - setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL") - setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL") - c.setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS") - setEnvString(&c.Storage.Retention.Default, "PROXY_STORAGE_RETENTION_DEFAULT") - setEnvString(&c.Storage.Retention.SweepInterval, "PROXY_STORAGE_RETENTION_SWEEP_INTERVAL") - setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER") - setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH") - setEnvString(&c.Database.URL, "PROXY_DATABASE_URL") - setEnvString(&c.Database.HitFlushInterval, "PROXY_DATABASE_HIT_FLUSH_INTERVAL") - setEnvString(&c.Log.Level, "PROXY_LOG_LEVEL") - setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT") - setEnvString(&c.AccessLog.Path, "PROXY_ACCESS_LOG_PATH") - setEnvStringSlice(&c.Upstream.AllowPrivateHosts, "PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS") - c.setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK") - setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM") - c.setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA") - c.setEnvBool(&c.Upstream.NPMChangeFeed, "PROXY_UPSTREAM_NPM_CHANGE_FEED") - c.setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE") - setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO") - setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD") - setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM") - setEnvString(&c.Upstream.Go, "PROXY_UPSTREAM_GO") - setEnvString(&c.Upstream.GoVulnDB, "PROXY_UPSTREAM_GO_VULNDB") - setEnvString(&c.Upstream.Hex, "PROXY_UPSTREAM_HEX") - setEnvString(&c.Upstream.HexAPI, "PROXY_UPSTREAM_HEX_API") - setEnvString(&c.Upstream.Pub, "PROXY_UPSTREAM_PUB") - setEnvString(&c.Upstream.PyPI, "PROXY_UPSTREAM_PYPI") - setEnvString(&c.Upstream.PyPIDownload, "PROXY_UPSTREAM_PYPI_DOWNLOAD") - setEnvString(&c.Upstream.Maven, "PROXY_UPSTREAM_MAVEN") - setEnvString(&c.Upstream.GradlePluginPortal, "PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL") - setEnvString(&c.Upstream.NuGet, "PROXY_UPSTREAM_NUGET") - setEnvString(&c.Upstream.NuGetSearch, "PROXY_UPSTREAM_NUGET_SEARCH") - setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER") - setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY") - c.setEnvBool(&c.Upstream.ComposerChangeFeed, "PROXY_UPSTREAM_COMPOSER_CHANGE_FEED") - setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN") - setEnvString(&c.Upstream.Chef, "PROXY_UPSTREAM_CHEF") - setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA") - setEnvString(&c.Upstream.CRAN, "PROXY_UPSTREAM_CRAN") - setEnvString(&c.Upstream.Julia, "PROXY_UPSTREAM_JULIA") - setEnvString(&c.Upstream.Swift, "PROXY_UPSTREAM_SWIFT") - setEnvString(&c.Upstream.OCIDefault, "PROXY_UPSTREAM_OCI_DEFAULT") - setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN") - setEnvString(&c.Upstream.APKSnapshots, "PROXY_UPSTREAM_APK_SNAPSHOTS") - setEnvString(&c.Upstream.RPM, "PROXY_UPSTREAM_RPM") - setEnvString(&c.Upstream.HomebrewAPI, "PROXY_UPSTREAM_HOMEBREW_API") - setEnvString(&c.Upstream.HomebrewArtifact, "PROXY_UPSTREAM_HOMEBREW_ARTIFACT") - setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT") - c.setEnvBool(&c.Scanning.Enabled, "PROXY_SCANNING_ENABLED") - c.setEnvBool(&c.Scanning.FailOpen, "PROXY_SCANNING_FAIL_OPEN") - setEnvString(&c.Scanning.Timeout, "PROXY_SCANNING_TIMEOUT") - setEnvString(&c.Scanning.SigningKey, "PROXY_SCANNING_SIGNING_KEY") - setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL") - c.setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") - c.setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") - c.setEnvBool(&c.CacheDelete, "PROXY_CACHE_DELETE") - c.setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") - setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN") - setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") - setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES") - setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS") - c.setEnvBool(&c.Projects.Enabled, "PROXY_PROJECTS_ENABLED") - setEnvString(&c.Projects.Token, "PROXY_PROJECTS_TOKEN") - setEnvString(&c.Projects.MaxAge, "PROXY_PROJECTS_MAX_AGE") - setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") - setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") - setEnvString(&c.MetadataStreamMaxSize, "PROXY_METADATA_STREAM_MAX_SIZE") - setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") - setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE") - setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT") - setEnvString(&c.WriteTimeout, "PROXY_WRITE_TIMEOUT") - setEnvString(&c.ArtifactTimeout, "PROXY_ARTIFACT_TIMEOUT") - c.setGradleReadOnlyFromEnv() - setEnvString(&c.Gradle.BuildCache.MaxUploadSize, "PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE") - setEnvString(&c.Gradle.BuildCache.MaxAge, "PROXY_GRADLE_BUILD_CACHE_MAX_AGE") - setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE") - setEnvString(&c.Gradle.BuildCache.SweepInterval, "PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL") - setEnvString(&c.Health.StorageProbeInterval, "PROXY_HEALTH_STORAGE_PROBE_INTERVAL") - setEnvString(&c.Git.CacheDir, "PROXY_GIT_CACHE_DIR") - setEnvString(&c.Git.MaxSize, "PROXY_GIT_MAX_SIZE") - setEnvString(&c.Git.MaxRepoSize, "PROXY_GIT_MAX_REPO_SIZE") - setEnvString(&c.Git.RefreshWait, "PROXY_GIT_REFRESH_WAIT") - setEnvString(&c.Git.StaleMaxAge, "PROXY_GIT_STALE_MAX_AGE") + if v := os.Getenv("PROXY_LISTEN"); v != "" { + c.Listen = v + } + if v := os.Getenv("PROXY_BASE_URL"); v != "" { + c.BaseURL = v + } + if v := os.Getenv("PROXY_UI_URL"); v != "" { + c.UIBaseURL = v + } + if v := os.Getenv("PROXY_STORAGE_URL"); v != "" { + c.Storage.URL = v + } + if v := os.Getenv("PROXY_STORAGE_PATH"); v != "" { + c.Storage.Path = v + } + if v := os.Getenv("PROXY_STORAGE_MAX_SIZE"); v != "" { + c.Storage.MaxSize = v + } + if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE"); v != "" { + c.Storage.DirectServe = envBool(v) + } + if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_TTL"); v != "" { + c.Storage.DirectServeTTL = v + } + if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_BASE_URL"); v != "" { + c.Storage.DirectServeBaseURL = v + } + if v := os.Getenv("PROXY_DATABASE_DRIVER"); v != "" { + c.Database.Driver = v + } + if v := os.Getenv("PROXY_DATABASE_PATH"); v != "" { + c.Database.Path = v + } + if v := os.Getenv("PROXY_DATABASE_URL"); v != "" { + c.Database.URL = v + } + if v := os.Getenv("PROXY_LOG_LEVEL"); v != "" { + c.Log.Level = v + } + if v := os.Getenv("PROXY_LOG_FORMAT"); v != "" { + c.Log.Format = v + } + if v := os.Getenv("PROXY_UPSTREAM_MAVEN"); v != "" { + c.Upstream.Maven = v + } + if v := os.Getenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL"); v != "" { + c.Upstream.GradlePluginPortal = v + } + if v := os.Getenv("PROXY_COOLDOWN_DEFAULT"); v != "" { + c.Cooldown.Default = v + } + if v := os.Getenv("PROXY_CACHE_METADATA"); v != "" { + c.CacheMetadata = envBool(v) + } + if v := os.Getenv("PROXY_MIRROR_API"); v != "" { + c.MirrorAPI = envBool(v) + } + if v := os.Getenv("PROXY_METADATA_TTL"); v != "" { + c.MetadataTTL = v + } + if v := os.Getenv("PROXY_METADATA_MAX_SIZE"); v != "" { + c.MetadataMaxSize = v + } + if v := os.Getenv("PROXY_HTTP_TIMEOUT"); v != "" { + c.HTTPTimeout = v + } + if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY"); v != "" { + c.Gradle.BuildCache.ReadOnly = v == "true" || v == "1" + } + if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE"); v != "" { + c.Gradle.BuildCache.MaxUploadSize = v + } + if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE"); v != "" { + c.Gradle.BuildCache.MaxAge = v + } + if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_SIZE"); v != "" { + c.Gradle.BuildCache.MaxSize = v + } + if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL"); v != "" { + c.Gradle.BuildCache.SweepInterval = v + } + if v := os.Getenv("PROXY_HEALTH_STORAGE_PROBE_INTERVAL"); v != "" { + c.Health.StorageProbeInterval = v + } } // validateAbsoluteURL returns an error if value is not a parseable URL with @@ -1680,8 +553,8 @@ func (c *Config) Validate() error { // Validate direct serve TTL if specified if c.Storage.DirectServeTTL != "" { - if _, err := parseNonNegativeDuration("storage.direct_serve_ttl", c.Storage.DirectServeTTL); err != nil { - return err + if _, err := time.ParseDuration(c.Storage.DirectServeTTL); err != nil { + return fmt.Errorf("invalid storage.direct_serve_ttl %q: %w", c.Storage.DirectServeTTL, err) } } @@ -1692,69 +565,18 @@ func (c *Config) Validate() error { } } - if err := c.validateCacheArtifacts(); err != nil { - return err - } - // Validate metadata TTL if specified if c.MetadataTTL != "" && c.MetadataTTL != "0" { - if _, err := parseNonNegativeDuration("metadata_ttl", c.MetadataTTL); err != nil { - return err + if _, err := time.ParseDuration(c.MetadataTTL); err != nil { + return fmt.Errorf("invalid metadata_ttl %q: %w", c.MetadataTTL, err) } } - if err := c.validateDownloadSizeLimits(); err != nil { + if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil { return err } - if err := c.validateRequestTimeouts(); err != nil { - return err - } - - if err := validateHitFlushInterval(c.Database.HitFlushInterval); err != nil { - return err - } - - return c.validateComponents() -} - -func (c *Config) validateCacheArtifacts() error { - if c.Storage.CacheArtifacts { - return nil - } - switch { - case c.Scanning.Enabled: - return fmt.Errorf("storage.cache_artifacts: false cannot be combined with scanning.enabled: scanning needs stored artifacts") - case c.Storage.DirectServe: - return fmt.Errorf("storage.cache_artifacts: false cannot be combined with storage.direct_serve: no artifacts are stored to redirect to") - case c.MirrorAPI: - return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served") - case len(c.Mirror.Schedules) > 0: - return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror.schedules: mirrored artifacts would never be served") - case c.Hosted.Enabled: - return fmt.Errorf("storage.cache_artifacts: false cannot be combined with hosted.enabled: pushed packages would never be served") - } - return nil -} - -func (c *Config) validateComponents() error { - if err := c.Cooldown.ValidateProfiles(); err != nil { - return err - } - - if err := validateMetadataRewriteCacheSize(c.MetadataRewriteCacheSize); err != nil { - return err - } - - if _, err := c.RetentionRules(retention.Default); err != nil { - return err - } - - if _, err := denylist.New(c.Denylist.Packages); err != nil { - return err - } - - if err := c.validateUpstream(); err != nil { + if err := validateHTTPTimeout(c.HTTPTimeout); err != nil { return err } @@ -1762,31 +584,11 @@ func (c *Config) validateComponents() error { return err } - if err := c.Scanning.Validate(); err != nil { + if err := c.Gradle.BuildCache.Validate(); err != nil { return err } - if err := c.Hosted.Validate(); err != nil { - return err - } - if _, reserved := c.Upstream.Generic[HostedGenericRepository]; reserved && c.Hosted.Enabled { - return fmt.Errorf("invalid upstream.generic name %q: reserved for pushed files at /generic/%s/ when hosted.enabled is set", - HostedGenericRepository, HostedGenericRepository) - } - - if err := c.Projects.Validate(); err != nil { - return err - } - - if err := c.Mirror.Validate(); err != nil { - return err - } - - if err := c.Git.Validate(); err != nil { - return err - } - - return c.Gradle.BuildCache.Validate() + return nil } // Validate checks the /health configuration. An unset interval is allowed @@ -1820,7 +622,7 @@ func (g *GradleBuildCacheConfig) Validate() error { } if g.MaxAge != "" && g.MaxAge != "0" { - if _, err := parseDayDuration(g.MaxAge); err != nil { + if _, err := time.ParseDuration(g.MaxAge); err != nil { return fmt.Errorf("invalid gradle.build_cache.max_age %q: %w", g.MaxAge, err) } } @@ -1847,22 +649,12 @@ func (g *GradleBuildCacheConfig) Validate() error { const ( defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default - defaultWriteTimeout = 5 * time.Minute - defaultArtifactTimeout = 5 * time.Minute defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default - defaultHitFlushInterval = time.Second defaultMetadataMaxSize = 100 << 20 - defaultMetadataStreamMaxSize = 1 << 30 - defaultMetadataRewriteCacheSize = 256 << 20 defaultGradleBuildCacheMaxUploadSize = 100 << 20 defaultGradleBuildCacheSweepInterval = 10 * time.Minute defaultGradleMaxUploadSizeStr = "100MB" defaultGradleSweepIntervalStr = "10m" - defaultRetentionSweepInterval = 10 * time.Minute - defaultRetentionSweepIntervalStr = "10m" - minRetentionSweepInterval = time.Minute - maxDurationDays = 36500 - defaultScanningTimeoutStr = "30s" ) // ParseMaxSize returns the maximum cache size in bytes. @@ -1878,102 +670,34 @@ func (c *Config) ParseMaxSize() int64 { return size } -func (c *Config) validateRequestTimeouts() error { - for _, timeout := range []struct{ field, value string }{ - {"http_timeout", c.HTTPTimeout}, - {"write_timeout", c.WriteTimeout}, - {"artifact_timeout", c.ArtifactTimeout}, - } { - if err := validateRequestTimeout(timeout.field, timeout.value); err != nil { - return err - } - } - - return nil -} - -func validateRequestTimeout(field, s string) error { +func validateHTTPTimeout(s string) error { if s == "" || s == "0" { return nil } d, err := time.ParseDuration(s) if err != nil { - return fmt.Errorf("invalid %s %q: %w", field, s, err) + return fmt.Errorf("invalid http_timeout %q: %w", s, err) } if d < 0 { - return fmt.Errorf("invalid %s %q: must be non-negative", field, s) + return fmt.Errorf("invalid http_timeout %q: must be non-negative", s) } return nil } -func validateMetadataMaxSize(name, s string) error { +func validateMetadataMaxSize(s string) error { if s == "" { return nil } size, err := ParseSize(s) if err != nil { - return fmt.Errorf("invalid %s: %w", name, err) + return fmt.Errorf("invalid metadata_max_size: %w", err) } if size <= 0 { - return fmt.Errorf("invalid %s %q: must be positive", name, s) + return fmt.Errorf("invalid metadata_max_size %q: must be positive", s) } return nil } -func (c *Config) validateDownloadSizeLimits() error { - if err := validateMetadataMaxSize("metadata_max_size", c.MetadataMaxSize); err != nil { - return err - } - if err := validateMetadataMaxSize("metadata_stream_max_size", c.MetadataStreamMaxSize); err != nil { - return err - } - return validateArtifactMaxSize(c.ArtifactMaxSize) -} - -func validateArtifactMaxSize(s string) error { - if s == "" { - return nil - } - size, err := ParseSize(s) - if err != nil { - return fmt.Errorf("invalid artifact_max_size: %w", err) - } - if size < 0 { - return fmt.Errorf("invalid artifact_max_size %q: must not be negative", s) - } - return nil -} - -func validateMetadataRewriteCacheSize(s string) error { - if s == "" || s == "0" { - return nil - } - size, err := ParseSize(s) - if err != nil { - return fmt.Errorf("invalid metadata_rewrite_cache_size: %w", err) - } - if size < 0 { - return fmt.Errorf("invalid metadata_rewrite_cache_size %q: must not be negative", s) - } - return nil -} - -// ParseMetadataRewriteCacheSize returns how many bytes of rewritten metadata -// to keep in memory. Returns 256MB if unset or invalid, 0 if disabled. -func (c *Config) ParseMetadataRewriteCacheSize() int64 { - if c.MetadataRewriteCacheSize == "" { - return defaultMetadataRewriteCacheSize - } - if c.MetadataRewriteCacheSize == "0" { - return 0 - } - size, err := ParseSize(c.MetadataRewriteCacheSize) - if err != nil || size < 0 { - return defaultMetadataRewriteCacheSize - } - return size -} - // ParseMetadataMaxSize returns the maximum metadata response size in bytes. // Returns 100MB if unset or invalid. func (c *Config) ParseMetadataMaxSize() int64 { @@ -1987,87 +711,18 @@ func (c *Config) ParseMetadataMaxSize() int64 { return size } -// ParseMetadataStreamMaxSize returns the size limit for metadata streamed -// into storage in bytes. Returns 1GB if unset or invalid. -func (c *Config) ParseMetadataStreamMaxSize() int64 { - if c.MetadataStreamMaxSize == "" { - return defaultMetadataStreamMaxSize - } - size, err := ParseSize(c.MetadataStreamMaxSize) - if err != nil || size <= 0 { - return defaultMetadataStreamMaxSize - } - return size -} - -// ParseArtifactMaxSize returns the artifact size limit in bytes, or 0 for -// no limit. -func (c *Config) ParseArtifactMaxSize() int64 { - if c.ArtifactMaxSize == "" { - return 0 - } - size, err := ParseSize(c.ArtifactMaxSize) - if err != nil || size < 0 { - return 0 - } - return size -} - // ParseHTTPTimeout returns the upstream HTTP client timeout. // Returns 30s if unset, 0 (no timeout) if explicitly set to "0". func (c *Config) ParseHTTPTimeout() time.Duration { - return parseRequestTimeout(c.HTTPTimeout, defaultHTTPTimeout) -} - -// ParseWriteTimeout returns the response write timeout, defaulting to 5m. -// An explicit zero disables the timeout. -func (c *Config) ParseWriteTimeout() time.Duration { - return parseRequestTimeout(c.WriteTimeout, defaultWriteTimeout) -} - -// ParseArtifactTimeout returns the upstream artifact download timeout for -// serving requests, defaulting to 5m. An explicit zero disables the timeout. -func (c *Config) ParseArtifactTimeout() time.Duration { - return parseRequestTimeout(c.ArtifactTimeout, defaultArtifactTimeout) -} - -func parseRequestTimeout(value string, fallback time.Duration) time.Duration { - if value == "" { - return fallback + if c.HTTPTimeout == "" { + return defaultHTTPTimeout } - d, err := time.ParseDuration(value) - if err != nil || d < 0 { - return fallback - } - return d -} - -func validateHitFlushInterval(s string) error { - if s == "" || s == "0" { - return nil - } - d, err := time.ParseDuration(s) - if err != nil { - return fmt.Errorf("invalid database.hit_flush_interval %q: %w", s, err) - } - if d < 0 { - return fmt.Errorf("invalid database.hit_flush_interval %q: must be non-negative", s) - } - return nil -} - -// ParseHitFlushInterval returns how often batched cache hits are written. -// Returns 1 second if unset or invalid, 0 if explicitly disabled. -func (c *Config) ParseHitFlushInterval() time.Duration { - if c.Database.HitFlushInterval == "" { - return defaultHitFlushInterval - } - if c.Database.HitFlushInterval == "0" { + if c.HTTPTimeout == "0" { return 0 } - d, err := time.ParseDuration(c.Database.HitFlushInterval) + d, err := time.ParseDuration(c.HTTPTimeout) if err != nil || d < 0 { - return defaultHitFlushInterval + return defaultHTTPTimeout } return d } @@ -2101,27 +756,13 @@ func (c *Config) ParseGradleBuildCacheMaxUploadSize() int64 { return size } -// ParseAPKSnapshots returns how long earlier Alpine APK indexes are kept -// after upstream replaced them. Returns 0 when snapshots are disabled or the -// value is invalid. -func (c *Config) ParseAPKSnapshots() time.Duration { - if c.Upstream.APKSnapshots == "" { - return 0 - } - d, err := parseDayDuration(c.Upstream.APKSnapshots) - if err != nil || d < 0 { - return 0 - } - return d -} - // ParseGradleBuildCacheMaxAge returns age-based eviction threshold. // Returns 0 when disabled or invalid. func (c *Config) ParseGradleBuildCacheMaxAge() time.Duration { if c.Gradle.BuildCache.MaxAge == "" || c.Gradle.BuildCache.MaxAge == "0" { return 0 } - d, err := parseDayDuration(c.Gradle.BuildCache.MaxAge) + d, err := time.ParseDuration(c.Gradle.BuildCache.MaxAge) if err != nil || d <= 0 { return 0 } @@ -2192,21 +833,13 @@ func ParseSize(s string) (int64, error) { } for _, s2 := range suffixes { - if numStr, ok := strings.CutSuffix(s, s2.suffix); ok { + if strings.HasSuffix(s, s2.suffix) { + numStr := strings.TrimSuffix(s, s2.suffix) num, err := strconv.ParseFloat(numStr, 64) if err != nil { return 0, fmt.Errorf("invalid number %q", numStr) } - // The comparisons also reject NaN. - size := num * float64(s2.mult) - if !(size >= 0 && size < math.MaxInt64) { - return 0, fmt.Errorf("invalid size %q: must be between 0 and %d bytes", s, int64(math.MaxInt64)) - } - // max_size 0 means unlimited, so don't let a tiny size become 0. - if num != 0 && int64(size) == 0 { - return 0, fmt.Errorf("invalid size %q: less than one byte", s) - } - return int64(size), nil + return int64(num * float64(s2.mult)), nil } } @@ -2215,9 +848,6 @@ func ParseSize(s string) (int64, error) { if err != nil { return 0, fmt.Errorf("invalid size %q", s) } - if num < 0 { - return 0, fmt.Errorf("invalid size %q: must not be negative", s) - } return num, nil } @@ -2258,3 +888,7 @@ func (a *AuthConfig) Header() (name, value string) { func expandEnv(s string) string { return os.Expand(s, os.Getenv) } + +func envBool(v string) bool { + return v == "true" || v == "1" +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ff8fc50..9c34023 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,8 +3,6 @@ package config import ( "os" "path/filepath" - "slices" - "strings" "testing" "time" ) @@ -15,115 +13,6 @@ const ( testLevelDebug = "debug" ) -func upstreamConfigValues(upstream UpstreamConfig) map[string]string { - return map[string]string{ - "npm": upstream.NPM, - "cargo": upstream.Cargo, - "cargo_download": upstream.CargoDownload, - "gem": upstream.Gem, - "go": upstream.Go, - "go_vulndb": upstream.GoVulnDB, - "hex": upstream.Hex, - "hex_api": upstream.HexAPI, - "pub": upstream.Pub, - "pypi": upstream.PyPI, - "pypi_download": upstream.PyPIDownload, - "maven": upstream.Maven, - "gradle_plugin_portal": upstream.GradlePluginPortal, - "nuget": upstream.NuGet, - "nuget_search": upstream.NuGetSearch, - "composer": upstream.Composer, - "composer_repository": upstream.ComposerRepository, - "conan": upstream.Conan, - "chef": upstream.Chef, - "conda": upstream.Conda, - "cran": upstream.CRAN, - "julia": upstream.Julia, - "swift": upstream.Swift, - "oci_default": upstream.OCIDefault, - "debian": upstream.Debian, - "rpm": upstream.RPM, - "homebrew_api": upstream.HomebrewAPI, - "homebrew_artifact": upstream.HomebrewArtifact, - } -} - -func defaultUpstreamValues() map[string]string { - return map[string]string{ - "npm": "https://registry.npmjs.org", - "cargo": "https://index.crates.io", - "cargo_download": "https://static.crates.io/crates", - "gem": "https://rubygems.org", - "go": "https://proxy.golang.org", - "go_vulndb": "https://vuln.go.dev", - "hex": "https://repo.hex.pm", - "hex_api": "https://hex.pm", - "pub": "https://pub.dev", - "pypi": "https://pypi.org", - "pypi_download": "https://files.pythonhosted.org", - "maven": "https://repo1.maven.org/maven2", - "gradle_plugin_portal": "https://plugins.gradle.org/m2", - "nuget": "https://api.nuget.org", - "nuget_search": "https://azuresearch-usnc.nuget.org", - "composer": "https://packagist.org", - "composer_repository": "https://repo.packagist.org", - "conan": "https://center.conan.io", - "chef": "https://supermarket.chef.io", - "conda": "https://conda.anaconda.org", - "cran": "https://cloud.r-project.org", - "julia": "https://pkg.julialang.org", - "swift": "https://tuist.dev/api/registry/swift", - "oci_default": "https://registry-1.docker.io", - "debian": "http://deb.debian.org/debian", - "rpm": "https://dl.fedoraproject.org/pub/fedora/linux", - "homebrew_api": "https://formulae.brew.sh/api", - "homebrew_artifact": "https://ghcr.io", - } -} - -func upstreamEnvironmentVariables() map[string]string { - return map[string]string{ - "npm": "PROXY_UPSTREAM_NPM", - "cargo": "PROXY_UPSTREAM_CARGO", - "cargo_download": "PROXY_UPSTREAM_CARGO_DOWNLOAD", - "gem": "PROXY_UPSTREAM_GEM", - "go": "PROXY_UPSTREAM_GO", - "go_vulndb": "PROXY_UPSTREAM_GO_VULNDB", - "hex": "PROXY_UPSTREAM_HEX", - "hex_api": "PROXY_UPSTREAM_HEX_API", - "pub": "PROXY_UPSTREAM_PUB", - "pypi": "PROXY_UPSTREAM_PYPI", - "pypi_download": "PROXY_UPSTREAM_PYPI_DOWNLOAD", - "maven": "PROXY_UPSTREAM_MAVEN", - "gradle_plugin_portal": "PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL", - "nuget": "PROXY_UPSTREAM_NUGET", - "nuget_search": "PROXY_UPSTREAM_NUGET_SEARCH", - "composer": "PROXY_UPSTREAM_COMPOSER", - "composer_repository": "PROXY_UPSTREAM_COMPOSER_REPOSITORY", - "conan": "PROXY_UPSTREAM_CONAN", - "chef": "PROXY_UPSTREAM_CHEF", - "conda": "PROXY_UPSTREAM_CONDA", - "cran": "PROXY_UPSTREAM_CRAN", - "julia": "PROXY_UPSTREAM_JULIA", - "swift": "PROXY_UPSTREAM_SWIFT", - "oci_default": "PROXY_UPSTREAM_OCI_DEFAULT", - "debian": "PROXY_UPSTREAM_DEBIAN", - "rpm": "PROXY_UPSTREAM_RPM", - "homebrew_api": "PROXY_UPSTREAM_HOMEBREW_API", - "homebrew_artifact": "PROXY_UPSTREAM_HOMEBREW_ARTIFACT", - } -} - -func assertUpstreamValues(t *testing.T, cfg *Config, want map[string]string) { - t.Helper() - got := upstreamConfigValues(cfg.Upstream) - for name, wantValue := range want { - if gotValue := got[name]; gotValue != wantValue { - t.Errorf("Upstream %s = %q, want %q", name, gotValue, wantValue) - } - } -} - func TestDefault(t *testing.T) { cfg := Default() @@ -136,25 +25,18 @@ func TestDefault(t *testing.T) { if cfg.Database.Path == "" { t.Error("Database.Path should not be empty") } - if cfg.AccessLog.Path != "" { - t.Errorf("AccessLog.Path = %q, want disabled by default", cfg.AccessLog.Path) - } if cfg.Gradle.BuildCache.MaxUploadSize != "100MB" { t.Errorf("Gradle.BuildCache.MaxUploadSize = %q, want %q", cfg.Gradle.BuildCache.MaxUploadSize, "100MB") } if cfg.Gradle.BuildCache.MaxAge != "168h" { t.Errorf("Gradle.BuildCache.MaxAge = %q, want %q", cfg.Gradle.BuildCache.MaxAge, "168h") } - if !cfg.Gradle.BuildCache.ReadOnly { - t.Error("Gradle.BuildCache.ReadOnly = false, want true by default") + if cfg.Upstream.Maven != "https://repo1.maven.org/maven2" { + t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://repo1.maven.org/maven2") } - if len(cfg.Upstream.AllowPrivateHosts) != 0 { - t.Errorf("Upstream.AllowPrivateHosts = %v, want empty", cfg.Upstream.AllowPrivateHosts) + if cfg.Upstream.GradlePluginPortal != "https://plugins.gradle.org/m2" { + t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.gradle.org/m2") } - if cfg.Upstream.AllowLoopback { - t.Error("Upstream.AllowLoopback = true, want false") - } - assertUpstreamValues(t, cfg, defaultUpstreamValues()) } func TestValidate(t *testing.T) { @@ -297,8 +179,6 @@ func TestParseSize(t *testing.T) { {"1TB", 1024 * 1024 * 1024 * 1024, false}, {"invalid", 0, true}, {"10XB", 0, true}, - {"0GB", 0, false}, - {"0.0MB", 0, false}, } for _, tt := range tests { @@ -328,11 +208,6 @@ database: log: level: "debug" format: "json" -access_log: - path: "/var/log/proxy/access.jsonl" -upstream: - homebrew_api: "https://homebrew-api.example.com" - homebrew_artifact: "https://homebrew-artifact.example.com" ` if err := os.WriteFile(path, []byte(content), 0644); err != nil { t.Fatalf("writing config file: %v", err) @@ -361,133 +236,6 @@ upstream: if cfg.Log.Format != "json" { t.Errorf("Log.Format = %q, want %q", cfg.Log.Format, "json") } - if cfg.AccessLog.Path != "/var/log/proxy/access.jsonl" { - t.Errorf("AccessLog.Path = %q, want %q", cfg.AccessLog.Path, "/var/log/proxy/access.jsonl") - } - if cfg.Upstream.HomebrewAPI != "https://homebrew-api.example.com" { - t.Errorf("Upstream.HomebrewAPI = %q, want %q", cfg.Upstream.HomebrewAPI, "https://homebrew-api.example.com") - } - if cfg.Upstream.HomebrewArtifact != "https://homebrew-artifact.example.com" { - t.Errorf("Upstream.HomebrewArtifact = %q, want %q", cfg.Upstream.HomebrewArtifact, "https://homebrew-artifact.example.com") - } -} - -func TestLoadYAMLUpstreams(t *testing.T) { - path := filepath.Join(t.TempDir(), "config.yaml") - content := ` -upstream: - allow_private_hosts: - - "registry.internal" - - "10.0.0.12" - allow_loopback: true - npm: "https://upstream.example.com/npm" - cargo: "https://upstream.example.com/cargo" - cargo_download: "https://upstream.example.com/cargo_download" - gem: "https://upstream.example.com/gem" - go: "https://upstream.example.com/go" - go_vulndb: "https://upstream.example.com/go_vulndb" - hex: "https://upstream.example.com/hex" - hex_api: "https://upstream.example.com/hex_api" - pub: "https://upstream.example.com/pub" - pypi: "https://upstream.example.com/pypi" - pypi_download: "https://upstream.example.com/pypi_download" - maven: "https://upstream.example.com/maven" - gradle_plugin_portal: "https://upstream.example.com/gradle_plugin_portal" - nuget: "https://upstream.example.com/nuget" - nuget_search: "https://upstream.example.com/nuget_search" - composer: "https://upstream.example.com/composer" - composer_repository: "https://upstream.example.com/composer_repository" - conan: "https://upstream.example.com/conan" - chef: "https://upstream.example.com/chef" - conda: "https://upstream.example.com/conda" - cran: "https://upstream.example.com/cran" - julia: "https://upstream.example.com/julia" - swift: "https://upstream.example.com/swift" - oci_default: "https://upstream.example.com/oci_default" - debian: "https://upstream.example.com/debian" - rpm: "https://upstream.example.com/rpm" - homebrew_api: "https://upstream.example.com/homebrew_api" - homebrew_artifact: "https://upstream.example.com/homebrew_artifact" -` - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatalf("writing config file: %v", err) - } - - cfg, err := Load(path) - if err != nil { - t.Fatalf("Load failed: %v", err) - } - - want := make(map[string]string) - for name := range defaultUpstreamValues() { - want[name] = "https://upstream.example.com/" + name - } - assertUpstreamValues(t, cfg, want) - if got := strings.Join(cfg.Upstream.AllowPrivateHosts, ","); got != "registry.internal,10.0.0.12" { - t.Errorf("Upstream.AllowPrivateHosts = %q, want %q", got, "registry.internal,10.0.0.12") - } - if !cfg.Upstream.AllowLoopback { - t.Error("Upstream.AllowLoopback = false, want true") - } -} - -func TestLoadGradleBuildCacheReadOnly(t *testing.T) { - tests := []struct { - name string - content string - want bool - }{ - {"block without read_only keeps default", "gradle:\n build_cache:\n max_age: \"12h\"\n", true}, - {"explicit false allows uploads", "gradle:\n build_cache:\n read_only: false\n", false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "config.yaml") - if err := os.WriteFile(path, []byte(tt.content), 0644); err != nil { - t.Fatalf("writing config file: %v", err) - } - cfg, err := Load(path) - if err != nil { - t.Fatalf("Load() error = %v", err) - } - if cfg.Gradle.BuildCache.ReadOnly != tt.want { - t.Errorf("Gradle.BuildCache.ReadOnly = %v, want %v", cfg.Gradle.BuildCache.ReadOnly, tt.want) - } - }) - } -} - -func TestLoadGradleBuildCacheReadOnlyFromEnv(t *testing.T) { - tests := []struct { - value string - fileRO bool - want bool - warnings int - }{ - {"false", true, false, 0}, - {"0", true, false, 0}, - {" FALSE ", true, false, 0}, - {"True", false, true, 0}, - {"1", false, true, 0}, - {"yes", false, true, 0}, - {"off", true, false, 0}, - {"maybe", false, true, 1}, - {"", false, false, 0}, - } - for _, tt := range tests { - t.Run(tt.value, func(t *testing.T) { - t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", tt.value) - cfg := Default() - cfg.Gradle.BuildCache.ReadOnly = tt.fileRO - cfg.LoadFromEnv() - if cfg.Gradle.BuildCache.ReadOnly != tt.want { - t.Errorf("Gradle.BuildCache.ReadOnly = %v, want %v", cfg.Gradle.BuildCache.ReadOnly, tt.want) - } - if got := len(cfg.EnvWarnings()); got != tt.warnings { - t.Errorf("len(EnvWarnings()) = %d, want %d: %v", got, tt.warnings, cfg.EnvWarnings()) - } - }) - } } func TestLoadJSON(t *testing.T) { @@ -496,11 +244,7 @@ func TestLoadJSON(t *testing.T) { content := `{ "listen": ":4000", - "base_url": "https://json.example.com", - "upstream": { - "gem": "https://json.example.com/gem", - "rpm": "https://json.example.com/rpm" - } + "base_url": "https://json.example.com" }` if err := os.WriteFile(path, []byte(content), 0644); err != nil { t.Fatalf("writing config file: %v", err) @@ -517,12 +261,6 @@ func TestLoadJSON(t *testing.T) { if cfg.BaseURL != "https://json.example.com" { t.Errorf("BaseURL = %q, want %q", cfg.BaseURL, "https://json.example.com") } - if cfg.Upstream.Gem != "https://json.example.com/gem" { - t.Errorf("Upstream.Gem = %q, want %q", cfg.Upstream.Gem, "https://json.example.com/gem") - } - if cfg.Upstream.RPM != "https://json.example.com/rpm" { - t.Errorf("Upstream.RPM = %q, want %q", cfg.Upstream.RPM, "https://json.example.com/rpm") - } } func TestLoadFromEnv(t *testing.T) { @@ -533,13 +271,9 @@ func TestLoadFromEnv(t *testing.T) { t.Setenv("PROXY_UI_URL", "https://ui.env.example.com/ui") t.Setenv("PROXY_STORAGE_PATH", "/env/cache") t.Setenv("PROXY_LOG_LEVEL", testLevelDebug) - t.Setenv("PROXY_ACCESS_LOG_PATH", "/tmp/proxy-access.jsonl") - t.Setenv("PROXY_TRUST_FORWARDED_FOR", "true") - t.Setenv("PROXY_UI_REQUEST_SOURCES", "true") - t.Setenv("PROXY_UI_FAILURE_DETAILS", "true") - t.Setenv("PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS", "registry.internal, 10.0.0.12") - t.Setenv("PROXY_UPSTREAM_ALLOW_LOOPBACK", "true") - t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "false") + t.Setenv("PROXY_UPSTREAM_MAVEN", "https://maven.example.com/repository/maven-public") + t.Setenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL", "https://plugins.example.com/m2") + t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE", "32MB") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE", "12h") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_SIZE", "10GB") @@ -562,36 +296,14 @@ func TestLoadFromEnv(t *testing.T) { if cfg.Log.Level != testLevelDebug { t.Errorf("Log.Level = %q, want %q", cfg.Log.Level, testLevelDebug) } - if cfg.AccessLog.Path != "/tmp/proxy-access.jsonl" { - t.Errorf("AccessLog.Path = %q, want %q", cfg.AccessLog.Path, "/tmp/proxy-access.jsonl") + if cfg.Upstream.Maven != "https://maven.example.com/repository/maven-public" { + t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://maven.example.com/repository/maven-public") } - // Containers configure the proxy through the environment, so a setting - // reachable only from YAML is unreachable in the deployment that most needs - // it -- which is the one sitting behind an ingress. - if !cfg.TrustForwardedFor { - t.Error("TrustForwardedFor = false, want true from the environment") + if cfg.Upstream.GradlePluginPortal != "https://plugins.example.com/m2" { + t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.example.com/m2") } - if !cfg.UIRequestSources { - t.Error("UIRequestSources = false, want true from the environment") - } - if !cfg.UIFailureDetails { - t.Error("UIFailureDetails = false, want true from the environment") - } - if got := strings.Join(cfg.Upstream.AllowPrivateHosts, ","); got != "registry.internal,10.0.0.12" { - t.Errorf("Upstream.AllowPrivateHosts = %q, want %q", got, "registry.internal,10.0.0.12") - } - if !cfg.Upstream.AllowLoopback { - t.Error("Upstream.AllowLoopback = false, want true") - } - - t.Setenv("PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS", " , ") - cfg.LoadFromEnv() - if got := strings.Join(cfg.Upstream.AllowPrivateHosts, ","); got != "registry.internal,10.0.0.12" { - t.Errorf("Upstream.AllowPrivateHosts after empty env = %q, want unchanged", got) - } - - if cfg.Gradle.BuildCache.ReadOnly { - t.Error("Gradle.BuildCache.ReadOnly = true, want false from the environment") + if !cfg.Gradle.BuildCache.ReadOnly { + t.Error("Gradle.BuildCache.ReadOnly = false, want true") } if cfg.Gradle.BuildCache.MaxUploadSize != "32MB" { t.Errorf("Gradle.BuildCache.MaxUploadSize = %q, want %q", cfg.Gradle.BuildCache.MaxUploadSize, "32MB") @@ -607,19 +319,6 @@ func TestLoadFromEnv(t *testing.T) { } } -func TestLoadFromEnvUpstreams(t *testing.T) { - cfg := Default() - want := make(map[string]string) - for name, envName := range upstreamEnvironmentVariables() { - value := "https://env.example.com/" + name - t.Setenv(envName, value) - want[name] = value - } - - cfg.LoadFromEnv() - assertUpstreamValues(t, cfg, want) -} - func TestLoadCooldownConfig(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "config.yaml") @@ -639,8 +338,6 @@ cooldown: packages: "pkg:npm/lodash": "0" "pkg:npm/@babel/core": "14d" - package_patterns: - "pkg:npm/@example/*": "0" ` if err := os.WriteFile(path, []byte(content), 0644); err != nil { t.Fatalf("writing config file: %v", err) @@ -666,37 +363,6 @@ cooldown: if cfg.Cooldown.Packages["pkg:npm/@babel/core"] != "14d" { t.Errorf("Cooldown.Packages[@babel/core] = %q, want %q", cfg.Cooldown.Packages["pkg:npm/@babel/core"], "14d") } - if got := cfg.Cooldown.NormalizedPackages()["pkg:npm/%40babel/core"]; got != "14d" { - t.Errorf("normalized Cooldown.Packages[@babel/core] = %q, want %q", got, "14d") - } - if cfg.Cooldown.PackagePatterns["pkg:npm/@example/*"] != "0" { - t.Errorf("Cooldown.PackagePatterns[example] = %q, want %q", cfg.Cooldown.PackagePatterns["pkg:npm/@example/*"], "0") - } -} - -func TestCooldownConfigNormalizedPackages(t *testing.T) { - rawScoped := "pkg:npm/@typescript/typescript-darwin-arm64" - canonicalScoped := "pkg:npm/%40typescript/typescript-darwin-arm64" - cfg := CooldownConfig{Packages: map[string]string{ - rawScoped: "2d", - canonicalScoped: "3d", - "not-a-purl": "4d", - }} - - got := cfg.NormalizedPackages() - - if got[canonicalScoped] != "3d" { - t.Errorf("canonical scoped package duration = %q, want %q", got[canonicalScoped], "3d") - } - if _, exists := got[rawScoped]; exists { - t.Errorf("raw scoped package key %q was not canonicalized", rawScoped) - } - if got["not-a-purl"] != "4d" { - t.Errorf("invalid PURL duration = %q, want preserved value %q", got["not-a-purl"], "4d") - } - if cfg.Packages[rawScoped] != "2d" { - t.Error("NormalizedPackages mutated the source map") - } } func TestLoadCooldownFromEnv(t *testing.T) { @@ -789,29 +455,6 @@ func TestParseMetadataMaxSize(t *testing.T) { } } -func TestParseMetadataStreamMaxSize(t *testing.T) { - cfg := Default() - if got := cfg.ParseMetadataStreamMaxSize(); got != defaultMetadataStreamMaxSize { - t.Errorf("unset: %d, want %d", got, defaultMetadataStreamMaxSize) - } - cfg.MetadataStreamMaxSize = "2GB" - if got := cfg.ParseMetadataStreamMaxSize(); got != 2<<30 { - t.Errorf("2GB: %d", got) - } - t.Setenv("PROXY_METADATA_STREAM_MAX_SIZE", "300MB") - cfg = Default() - cfg.LoadFromEnv() - if got := cfg.ParseMetadataStreamMaxSize(); got != 300<<20 { - t.Errorf("from env: %d", got) - } - for _, bad := range []string{"0", "lots"} { - cfg.MetadataStreamMaxSize = bad - if err := cfg.Validate(); err == nil { - t.Errorf("metadata_stream_max_size %q validated", bad) - } - } -} - func TestValidateMetadataMaxSize(t *testing.T) { cfg := Default() cfg.MetadataMaxSize = "not-a-size" @@ -881,117 +524,6 @@ func TestValidateHealthStorageProbeInterval(t *testing.T) { } } -func TestScanningConfigValidate(t *testing.T) { - tests := []struct { - name string - cfg ScanningConfig - wantErr bool - }{ - { - name: "disabled skips validation entirely", - cfg: ScanningConfig{Enabled: false, Timeout: "not-a-duration"}, - }, - { - name: "enabled without signing key fails", - cfg: ScanningConfig{Enabled: true}, - wantErr: true, - }, - { - name: "enabled with signing key and no scanners fails", - cfg: ScanningConfig{Enabled: true, SigningKey: "s3cret"}, - wantErr: true, - }, - { - name: "invalid fetch_base_url fails", - cfg: ScanningConfig{ - Enabled: true, - SigningKey: "s3cret", - FetchBaseURL: "not-a-url", - Scanners: []ScannerConfig{{Name: "clamav", URL: "http://scanner.invalid/scan"}}, - }, - wantErr: true, - }, - { - name: "invalid timeout fails", - cfg: ScanningConfig{ - Enabled: true, - SigningKey: "s3cret", - Timeout: "not-a-duration", - Scanners: []ScannerConfig{{Name: "clamav", URL: "http://scanner.invalid/scan"}}, - }, - wantErr: true, - }, - { - name: "zero timeout fails", - cfg: ScanningConfig{ - Enabled: true, - SigningKey: "s3cret", - Timeout: "0", - Scanners: []ScannerConfig{{Name: "clamav", URL: "http://scanner.invalid/scan"}}, - }, - wantErr: true, - }, - { - name: "empty timeout defaults and is valid", - cfg: ScanningConfig{ - Enabled: true, - SigningKey: "s3cret", - Timeout: "", - Scanners: []ScannerConfig{{Name: "clamav", URL: "http://scanner.invalid/scan"}}, - }, - }, - { - name: "scanner missing name fails", - cfg: ScanningConfig{ - Enabled: true, - SigningKey: "s3cret", - Scanners: []ScannerConfig{{URL: "http://scanner.invalid/scan"}}, - }, - wantErr: true, - }, - { - name: "scanner with invalid url fails", - cfg: ScanningConfig{ - Enabled: true, - SigningKey: "s3cret", - Scanners: []ScannerConfig{{Name: "clamav", URL: "not-a-url"}}, - }, - wantErr: true, - }, - { - name: "scanner with invalid mode fails", - cfg: ScanningConfig{ - Enabled: true, - SigningKey: "s3cret", - Scanners: []ScannerConfig{ - {Name: "clamav", URL: "http://scanner.invalid/scan", Mode: "quarantine"}, - }, - }, - wantErr: true, - }, - { - name: "scanner with default mode is valid", - cfg: ScanningConfig{ - Enabled: true, - SigningKey: "s3cret", - Scanners: []ScannerConfig{{Name: "clamav", URL: "http://scanner.invalid/scan"}}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := tt.cfg.Validate() - if tt.wantErr && err == nil { - t.Error("Validate() error = nil, want error") - } - if !tt.wantErr && err != nil { - t.Errorf("Validate() unexpected error: %v", err) - } - }) - } -} - func TestParseHTTPTimeout(t *testing.T) { tests := []struct { name string @@ -1045,54 +577,6 @@ func TestValidateHTTPTimeout(t *testing.T) { } } -func TestParseMetadataRewriteCacheSize(t *testing.T) { - tests := []struct { - name string - size string - want int64 - }{ - {"empty defaults to 256MB", "", 256 << 20}, - {"explicit zero disables", "0", 0}, - {"1GB", "1GB", 1 << 30}, - {"invalid defaults to 256MB", "lots", 256 << 20}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := Default() - cfg.MetadataRewriteCacheSize = tt.size - if got := cfg.ParseMetadataRewriteCacheSize(); got != tt.want { - t.Errorf("ParseMetadataRewriteCacheSize() = %d, want %d", got, tt.want) - } - }) - } -} - -func TestValidateMetadataRewriteCacheSize(t *testing.T) { - cfg := Default() - cfg.MetadataRewriteCacheSize = "lots" - if err := cfg.Validate(); err == nil { - t.Error("expected validation error for invalid metadata_rewrite_cache_size") - } - - for _, ok := range []string{"512MB", "0", ""} { - cfg.MetadataRewriteCacheSize = ok - if err := cfg.Validate(); err != nil { - t.Errorf("unexpected error for metadata_rewrite_cache_size %q: %v", ok, err) - } - } -} - -func TestLoadMetadataRewriteCacheSizeFromEnv(t *testing.T) { - cfg := Default() - t.Setenv("PROXY_METADATA_REWRITE_CACHE_SIZE", "1GB") - cfg.LoadFromEnv() - - if cfg.MetadataRewriteCacheSize != "1GB" { - t.Errorf("MetadataRewriteCacheSize = %q, want %q", cfg.MetadataRewriteCacheSize, "1GB") - } -} - func TestLoadHTTPTimeoutFromEnv(t *testing.T) { cfg := Default() t.Setenv("PROXY_HTTP_TIMEOUT", "90s") @@ -1103,61 +587,6 @@ func TestLoadHTTPTimeoutFromEnv(t *testing.T) { } } -func TestParseHitFlushInterval(t *testing.T) { - tests := []struct { - name string - interval string - want time.Duration - }{ - {"empty defaults to 1s", "", time.Second}, - {"explicit zero disables", "0", 0}, - {"10 seconds", "10s", 10 * time.Second}, - {"invalid defaults to 1s", "not-a-duration", time.Second}, - {"negative defaults to 1s", "-5s", time.Second}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := Default() - cfg.Database.HitFlushInterval = tt.interval - got := cfg.ParseHitFlushInterval() - if got != tt.want { - t.Errorf("ParseHitFlushInterval() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestValidateHitFlushInterval(t *testing.T) { - cfg := Default() - cfg.Database.HitFlushInterval = "not-a-duration" - if err := cfg.Validate(); err == nil { - t.Error("expected validation error for invalid hit_flush_interval") - } - - cfg.Database.HitFlushInterval = "-5s" - if err := cfg.Validate(); err == nil { - t.Error("expected validation error for negative hit_flush_interval") - } - - for _, ok := range []string{"10s", "0", ""} { - cfg.Database.HitFlushInterval = ok - if err := cfg.Validate(); err != nil { - t.Errorf("unexpected error for hit_flush_interval %q: %v", ok, err) - } - } -} - -func TestLoadHitFlushIntervalFromEnv(t *testing.T) { - cfg := Default() - t.Setenv("PROXY_DATABASE_HIT_FLUSH_INTERVAL", "10s") - cfg.LoadFromEnv() - - if cfg.Database.HitFlushInterval != "10s" { - t.Errorf("HitFlushInterval = %q, want %q", cfg.Database.HitFlushInterval, "10s") - } -} - func TestLoadMetadataTTLFromEnv(t *testing.T) { cfg := Default() t.Setenv("PROXY_METADATA_TTL", "10m") @@ -1168,17 +597,6 @@ func TestLoadMetadataTTLFromEnv(t *testing.T) { } } -func TestLoadChangeFeedsFromEnv(t *testing.T) { - cfg := Default() - t.Setenv("PROXY_UPSTREAM_NPM_CHANGE_FEED", "true") - t.Setenv("PROXY_UPSTREAM_COMPOSER_CHANGE_FEED", "true") - cfg.LoadFromEnv() - - if !cfg.Upstream.NPMChangeFeed || !cfg.Upstream.ComposerChangeFeed { - t.Errorf("change feeds = npm %t, composer %t, want both on", cfg.Upstream.NPMChangeFeed, cfg.Upstream.ComposerChangeFeed) - } -} - func TestParseGradleBuildCacheConfig(t *testing.T) { cfg := Default() @@ -1342,932 +760,3 @@ func TestDatabaseConfigString(t *testing.T) { } } } - -func TestUpstreamAuthForURLMatchesURLComponents(t *testing.T) { - registryAuth := AuthConfig{Type: "bearer", Token: "registry-token"} - privateAuth := AuthConfig{Type: "bearer", Token: "private-token"} - config := UpstreamConfig{Auth: map[string]AuthConfig{ - "https://registry.example.com": registryAuth, - "https://registry.example.com/private": privateAuth, - }} - - tests := []struct { - name string - url string - wantToken string - }{ - {name: "registry root", url: "https://registry.example.com/package", wantToken: "registry-token"}, - {name: "host is case insensitive", url: "https://REGISTRY.EXAMPLE.COM/package", wantToken: "registry-token"}, - {name: "longest path match", url: "https://registry.example.com/private/package", wantToken: "private-token"}, - {name: "exact path match", url: "https://registry.example.com/private", wantToken: "private-token"}, - {name: "path segment boundary", url: "https://registry.example.com/private-other/package", wantToken: "registry-token"}, - {name: "lookalike host rejected", url: "https://registry.example.com.evil.test/package"}, - {name: "different scheme rejected", url: "http://registry.example.com/package"}, - {name: "different port rejected", url: "https://registry.example.com:8443/package"}, - {name: "dot dot segment rejected", url: "https://registry.example.com/private/../package"}, - {name: "encoded dot dot segment rejected", url: "https://registry.example.com/private/%2e%2e/package"}, - {name: "double encoded dot dot segment rejected", url: "https://registry.example.com/private/%252e%252e/package"}, - {name: "dot segment rejected", url: "https://registry.example.com/private/./package"}, - {name: "dot dot segment with parameter rejected", url: "https://registry.example.com/private/..;/package"}, - {name: "encoded dot dot segment with parameter rejected", url: "https://registry.example.com/private/%2e%2e;x/package"}, - {name: "dot dot segment with encoded parameter rejected", url: "https://registry.example.com/private/..%3b/package"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - auth := config.AuthForURL(tt.url) - if tt.wantToken == "" { - if auth != nil { - t.Fatalf("AuthForURL() = %+v, want nil", auth) - } - return - } - if auth == nil { - t.Fatal("AuthForURL() = nil, want authentication") - } - if auth.Token != tt.wantToken { - t.Errorf("token = %q, want %q", auth.Token, tt.wantToken) - } - }) - } -} - -func TestValidateUpstreamAuthURLs(t *testing.T) { - t.Run("valid absolute URL", func(t *testing.T) { - cfg := Default() - cfg.Upstream.Auth = map[string]AuthConfig{ - "https://registry.example.com/private": {Type: "bearer", Token: "token"}, - } - - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate() error = %v", err) - } - }) - - t.Run("invalid URL", func(t *testing.T) { - cfg := Default() - cfg.Upstream.Auth = map[string]AuthConfig{ - "registry.example.com": {Type: "bearer", Token: "token"}, - } - - err := cfg.Validate() - if err == nil { - t.Fatal("Validate() error = nil, want invalid upstream.auth URL error") - } - if !strings.Contains(err.Error(), "upstream.auth") || !strings.Contains(err.Error(), "registry.example.com") { - t.Errorf("Validate() error = %q, want field and URL", err) - } - }) -} - -func TestValidatePackageRoutes(t *testing.T) { - tests := []struct { - name string - modify func(*Config) - wantErr bool - }{ - { - name: "valid Composer and npm routes", - modify: func(cfg *Config) { - cfg.Upstream.ComposerRoutes = map[string]string{"acme/*": "https://composer.example.com/packages"} - cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": "https://npm.example.com/registry"} - }, - }, - { - name: "Composer route URL is not absolute", - modify: func(cfg *Config) { - cfg.Upstream.ComposerRoutes = map[string]string{"acme/*": "composer.example.com"} - }, - wantErr: true, - }, - { - name: "npm route URL is not absolute", - modify: func(cfg *Config) { - cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": "npm.example.com"} - }, - wantErr: true, - }, - { - name: "Composer route pattern is malformed", - modify: func(cfg *Config) { - cfg.Upstream.ComposerRoutes = map[string]string{"acme/[": "https://composer.example.com"} - }, - wantErr: true, - }, - { - name: "npm route pattern is empty", - modify: func(cfg *Config) { - cfg.Upstream.NPMRoutes = map[string]string{"": "https://npm.example.com"} - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := Default() - tt.modify(cfg) - err := cfg.Validate() - if (err != nil) != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %t", err, tt.wantErr) - } - }) - } -} - -func TestValidateNamedUpstreams(t *testing.T) { - tests := []struct { - name string - modify func(*Config) - wantErr bool - }{ - { - name: "valid Helm, OCI, APK, Arch, and generic upstreams", - modify: func(cfg *Config) { - cfg.Upstream.Helm = map[string]string{"bitnami": "https://charts.bitnami.com/bitnami"} - cfg.Upstream.OCI = map[string]string{"ghcr": "https://ghcr.io"} - cfg.Upstream.APK = map[string]string{"alpine": "https://dl-cdn.alpinelinux.org/alpine"} - cfg.Upstream.Arch = map[string]string{ - "archlinux": "https://geo.mirror.pkgbuild.com", - "archlinuxarm": "https://de3.mirror.archlinuxarm.org", - } - cfg.Upstream.Generic = map[string]string{ - "github": "https://github.com", - "github-api": "https://api.github.com", - } - }, - }, - { - name: "generic upstream name contains path separator", - modify: func(cfg *Config) { - cfg.Upstream.Generic = map[string]string{"github/releases": "https://github.com"} - }, - wantErr: true, - }, - { - name: "generic artifact patterns", - modify: func(cfg *Config) { - cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"} - cfg.Upstream.GenericArtifacts = map[string][]string{ - "apache": {`dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)`}, - } - }, - }, - { - name: "generic artifact patterns for an unknown upstream", - modify: func(cfg *Config) { - cfg.Upstream.GenericArtifacts = map[string][]string{ - "apache": {`(?P[^/]+)/(?P[^/]+)`}, - } - }, - wantErr: true, - }, - { - name: "generic artifact pattern does not compile", - modify: func(cfg *Config) { - cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"} - cfg.Upstream.GenericArtifacts = map[string][]string{"apache": {`(?P[^/]+/(?P[^/]+)`}} - }, - wantErr: true, - }, - { - name: "generic artifact pattern without a file group", - modify: func(cfg *Config) { - cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"} - cfg.Upstream.GenericArtifacts = map[string][]string{"apache": {`(?P[^/]+)/[^/]+`}} - }, - wantErr: true, - }, - { - name: "generic upstream URL is not absolute", - modify: func(cfg *Config) { - cfg.Upstream.Generic = map[string]string{"github": "github.com"} - }, - wantErr: true, - }, - { - name: "Helm upstream name contains path separator", - modify: func(cfg *Config) { - cfg.Upstream.Helm = map[string]string{"team/charts": "https://charts.example.com"} - }, - wantErr: true, - }, - { - name: "OCI mirrors for a configured upstream", - modify: func(cfg *Config) { - cfg.Upstream.OCI = map[string]string{"ghcr": "https://artifactory.example/api/docker/ghcr-remote"} - cfg.Upstream.OCIMirrors = map[string][]string{"ghcr": {"ghcr.io", "registry.example:5000"}} - }, - }, - { - name: "OCI mirrors for an unknown upstream", - modify: func(cfg *Config) { - cfg.Upstream.OCI = map[string]string{"ghcr": "https://ghcr.io"} - cfg.Upstream.OCIMirrors = map[string][]string{"quay": {"quay.io"}} - }, - wantErr: true, - }, - { - name: "OCI mirror given as URL", - modify: func(cfg *Config) { - cfg.Upstream.OCI = map[string]string{"ghcr": "https://mirror.example"} - cfg.Upstream.OCIMirrors = map[string][]string{"ghcr": {"https://ghcr.io"}} - }, - wantErr: true, - }, - { - name: "OCI mirror with a path", - modify: func(cfg *Config) { - cfg.Upstream.OCI = map[string]string{"ghcr": "https://mirror.example"} - cfg.Upstream.OCIMirrors = map[string][]string{"ghcr": {"ghcr.io/owner"}} - }, - wantErr: true, - }, - { - name: "OCI mirror host with credentials", - modify: func(cfg *Config) { - cfg.Upstream.OCI = map[string]string{"ghcr": "https://mirror.example"} - cfg.Upstream.OCIMirrors = map[string][]string{"ghcr": {"user@ghcr.io"}} - }, - wantErr: true, - }, - { - name: "OCI mirror host listed for two upstreams", - modify: func(cfg *Config) { - cfg.Upstream.OCI = map[string]string{"art": "https://art.example", "nexus": "https://nexus.example"} - cfg.Upstream.OCIMirrors = map[string][]string{"art": {"ghcr.io"}, "nexus": {"GHCR.io:443"}} - }, - wantErr: true, - }, - { - name: "OCI mirror host listed for two upstreams with an empty port", - modify: func(cfg *Config) { - cfg.Upstream.OCI = map[string]string{"art": "https://art.example", "nexus": "https://nexus.example"} - cfg.Upstream.OCIMirrors = map[string][]string{"art": {"ghcr.io"}, "nexus": {"ghcr.io:"}} - }, - wantErr: true, - }, - { - name: "OCI mirror IPv6 host listed for two upstreams", - modify: func(cfg *Config) { - cfg.Upstream.OCI = map[string]string{"art": "https://art.example", "nexus": "https://nexus.example"} - cfg.Upstream.OCIMirrors = map[string][]string{"art": {"[::1]"}, "nexus": {"[::1]:"}} - }, - wantErr: true, - }, - { - name: "OCI mirror hosts that differ only in a non-default port", - modify: func(cfg *Config) { - cfg.Upstream.OCI = map[string]string{"art": "https://art.example", "nexus": "https://nexus.example"} - cfg.Upstream.OCIMirrors = map[string][]string{"art": {"registry.example"}, "nexus": {"registry.example:5000"}} - }, - }, - { - name: "OCI mirror host is empty", - modify: func(cfg *Config) { - cfg.Upstream.OCI = map[string]string{"ghcr": "https://mirror.example"} - cfg.Upstream.OCIMirrors = map[string][]string{"ghcr": {""}} - }, - wantErr: true, - }, - { - name: "OCI upstream URL is not absolute", - modify: func(cfg *Config) { - cfg.Upstream.OCI = map[string]string{"private": "registry.example.com"} - }, - wantErr: true, - }, - { - name: "APK upstream name contains path separator", - modify: func(cfg *Config) { - cfg.Upstream.APK = map[string]string{"alpine/edge": "https://dl-cdn.alpinelinux.org/alpine"} - }, - wantErr: true, - }, - { - name: "APK upstream URL is not absolute", - modify: func(cfg *Config) { - cfg.Upstream.APK = map[string]string{"private": "apk.example.com"} - }, - wantErr: true, - }, - { - name: "Arch upstream name contains path separator", - modify: func(cfg *Config) { - cfg.Upstream.Arch = map[string]string{"arch/linux": "https://geo.mirror.pkgbuild.com"} - }, - wantErr: true, - }, - { - name: "Arch upstream URL is not absolute", - modify: func(cfg *Config) { - cfg.Upstream.Arch = map[string]string{"archlinux": "geo.mirror.pkgbuild.com"} - }, - wantErr: true, - }, - { - name: "valid Debian repository", - modify: func(cfg *Config) { - cfg.Upstream.DebianRepositories = map[string]string{ - "security": "https://security.debian.org/debian-security", - } - }, - }, - { - name: "Debian repository name contains path separator", - modify: func(cfg *Config) { - cfg.Upstream.DebianRepositories = map[string]string{ - "debian/security": "https://security.debian.org/debian-security", - } - }, - wantErr: true, - }, - { - name: "Debian repository name is a relative path element", - modify: func(cfg *Config) { - cfg.Upstream.DebianRepositories = map[string]string{ - ".": "https://security.debian.org/debian-security", - } - }, - wantErr: true, - }, - { - name: "Debian repository URL is not absolute", - modify: func(cfg *Config) { - cfg.Upstream.DebianRepositories = map[string]string{"security": "security.debian.org"} - }, - wantErr: true, - }, - { - name: "Debian repository shadows the pool prefix", - modify: func(cfg *Config) { - cfg.Upstream.DebianRepositories = map[string]string{"pool": "https://security.debian.org"} - }, - wantErr: true, - }, - { - name: "Debian repository shadows the dists prefix", - modify: func(cfg *Config) { - cfg.Upstream.DebianRepositories = map[string]string{"dists": "https://security.debian.org"} - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := Default() - tt.modify(cfg) - err := cfg.Validate() - if (err != nil) != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %t", err, tt.wantErr) - } - }) - } -} - -func TestValidateGoSumDBs(t *testing.T) { - tests := []struct { - name string - modify func(*Config) - wantErr bool - }{ - { - name: "valid Go checksum database", - modify: func(cfg *Config) { - cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org": "https://sum.golang.org"} - }, - }, - { - name: "Go checksum database name contains path separator", - modify: func(cfg *Config) { - cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org/x": "https://sum.golang.org"} - }, - wantErr: true, - }, - { - name: "Go checksum database URL is not absolute", - modify: func(cfg *Config) { - cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org": "sum.golang.org"} - }, - wantErr: true, - }, - { - name: "Go checksum database offline mode requires metadata caching", - modify: func(cfg *Config) { - cfg.CacheMetadata = false - cfg.Upstream.GoSumDBsOffline = true - }, - wantErr: true, - }, - { - name: "Go checksum database offline mode with metadata caching", - modify: func(cfg *Config) { - cfg.CacheMetadata = true - cfg.Upstream.GoSumDBsOffline = true - }, - }, - { - name: "npm change feed requires metadata caching", - modify: func(cfg *Config) { - cfg.CacheMetadata = false - cfg.Upstream.NPMChangeFeed = true - }, - wantErr: true, - }, - { - name: "Composer change feed requires metadata caching", - modify: func(cfg *Config) { - cfg.CacheMetadata = false - cfg.Upstream.ComposerChangeFeed = true - }, - wantErr: true, - }, - { - name: "change feeds with metadata caching", - modify: func(cfg *Config) { - cfg.CacheMetadata = true - cfg.Upstream.NPMChangeFeed = true - cfg.Upstream.ComposerChangeFeed = true - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := Default() - tt.modify(cfg) - err := cfg.Validate() - if (err != nil) != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %t", err, tt.wantErr) - } - }) - } -} - -func TestValidateCacheArtifactsDisabled(t *testing.T) { - tests := []struct { - name string - modify func(*Config) - wantErr string - }{ - {"alone", func(*Config) {}, ""}, - {"with scanning", func(c *Config) { c.Scanning.Enabled = true }, "scanning.enabled"}, - {"with direct_serve", func(c *Config) { c.Storage.DirectServe = true }, "storage.direct_serve"}, - {"with mirror_api", func(c *Config) { c.MirrorAPI = true }, "mirror_api"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := Default() - cfg.Storage.CacheArtifacts = false - tt.modify(cfg) - err := cfg.Validate() - if tt.wantErr == "" { - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - return - } - if err == nil || !strings.Contains(err.Error(), tt.wantErr) { - t.Fatalf("Validate() = %v, want an error mentioning %q", err, tt.wantErr) - } - }) - } -} - -func TestCacheArtifactsDefaultsToTrue(t *testing.T) { - if !Default().Storage.CacheArtifacts { - t.Fatal("Default().Storage.CacheArtifacts = false, want true") - } - - path := filepath.Join(t.TempDir(), "config.yaml") - if err := os.WriteFile(path, []byte("storage:\n url: \"file:///tmp/cache\"\n"), 0o600); err != nil { - t.Fatal(err) - } - cfg, err := Load(path) - if err != nil { - t.Fatalf("Load: %v", err) - } - if !cfg.Storage.CacheArtifacts { - t.Error("a config file without storage.cache_artifacts disabled artifact caching") - } -} - -func TestLoadCacheArtifactsFromEnv(t *testing.T) { - cfg := Default() - t.Setenv("PROXY_STORAGE_CACHE_ARTIFACTS", "false") - cfg.LoadFromEnv() - - if cfg.Storage.CacheArtifacts { - t.Error("Storage.CacheArtifacts should be false") - } -} - -func TestParseArtifactMaxSize(t *testing.T) { - tests := []struct { - name string - size string - want int64 - }{ - {"unset is unlimited", "", 0}, - {"zero is unlimited", "0", 0}, - {"explicit value", "2GB", 2 << 30}, - {"bytes", "1024", 1024}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := Default() - cfg.ArtifactMaxSize = tt.size - if got := cfg.ParseArtifactMaxSize(); got != tt.want { - t.Errorf("ParseArtifactMaxSize() = %d, want %d", got, tt.want) - } - }) - } -} - -func TestValidateArtifactMaxSize(t *testing.T) { - for _, invalid := range []string{"not-a-size", "-1"} { - cfg := Default() - cfg.ArtifactMaxSize = invalid - if err := cfg.Validate(); err == nil { - t.Errorf("expected validation error for artifact_max_size %q", invalid) - } - } - for _, valid := range []string{"", "0", "2GB"} { - cfg := Default() - cfg.ArtifactMaxSize = valid - if err := cfg.Validate(); err != nil { - t.Errorf("unexpected error for artifact_max_size %q: %v", valid, err) - } - } -} - -func TestLoadArtifactMaxSizeFromEnv(t *testing.T) { - cfg := Default() - t.Setenv("PROXY_ARTIFACT_MAX_SIZE", "2GB") - cfg.LoadFromEnv() - - if cfg.ArtifactMaxSize != "2GB" { - t.Errorf("ArtifactMaxSize = %q, want %q", cfg.ArtifactMaxSize, "2GB") - } -} - -func TestMavenRepositoryNames(t *testing.T) { - for _, tt := range []struct { - name string - wantErr bool - }{ - {"grails", false}, - {"grails-restricted", false}, - {"company-releases", false}, - {"org", true}, - {"com", true}, - {"tools", true}, - {"software", true}, - {"jakarta", true}, - {"javax", true}, - {"junit", true}, - {"androidx", true}, - {"gradle", true}, - {"nebula", true}, - {"my repo", true}, - {"rel#1", true}, - {"rel?x", true}, - {"-releases", true}, - {"repo_2.x", false}, - {"ORG", false}, - {"Tools", false}, - {"io", true}, - {"IO", true}, - {"de", true}, - {"x", true}, - {"net", true}, - {"a/b", true}, - } { - cfg := Default() - cfg.Upstream.MavenRepositories = map[string]string{tt.name: "https://repo.example/maven"} - err := cfg.Validate() - if (err != nil) != tt.wantErr { - t.Errorf("maven_repositories name %q: error = %v, want error %v", tt.name, err, tt.wantErr) - } - } - - cfg := Default() - cfg.Upstream.MavenRepositories = map[string]string{"grails": "not a url"} - if err := cfg.Validate(); err == nil { - t.Error("maven_repositories with an invalid URL: want an error") - } -} - -func TestGenericUpstreamNamedHosted(t *testing.T) { - cfg := Default() - cfg.Upstream.Generic = map[string]string{"hosted": "https://example.com"} - if err := cfg.Validate(); err != nil { - t.Errorf("upstream.generic named hosted with hosting off: %v, want it accepted", err) - } - - cfg = Default() - cfg.Upstream.Generic = map[string]string{"hosted": "https://example.com"} - cfg.Hosted.Enabled = true - cfg.Hosted.Token = "secret" - if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "reserved") { - t.Errorf("upstream.generic named hosted with hosting on: %v, want it refused", err) - } -} - -func TestValidateFreeBSDUpstreams(t *testing.T) { - tests := []struct { - name string - freebsd map[string]string - wantErr bool - }{ - {"valid", map[string]string{"freebsd": "https://pkg.freebsd.org", "poudriere": "https://packages.example.com"}, false}, - {"name contains path separator", map[string]string{"freebsd/ports": "https://pkg.freebsd.org"}, true}, - {"URL is not absolute", map[string]string{"freebsd": "pkg.freebsd.org"}, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := Default() - cfg.Upstream.FreeBSD = tt.freebsd - if err := cfg.Validate(); (err != nil) != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestValidateGenericPaths(t *testing.T) { - githubAPI := map[string]string{"github-api": "https://api.github.com"} - token := map[string]AuthConfig{"https://api.github.com": {Type: "bearer", Token: "t"}} - tests := []struct { - name string - upstream UpstreamConfig - wantErr bool - }{ - {name: "public", upstream: UpstreamConfig{Generic: githubAPI}}, - {name: "credentials without paths", upstream: UpstreamConfig{Generic: githubAPI, Auth: token}, wantErr: true}, - { - name: "credentials for a path under the upstream", - upstream: UpstreamConfig{ - Generic: githubAPI, - Auth: map[string]AuthConfig{"https://api.github.com/repos/acme": {Type: "bearer", Token: "t"}}, - }, - wantErr: true, - }, - { - name: "credentials with paths", - upstream: UpstreamConfig{ - Generic: githubAPI, - GenericPaths: map[string][]string{"github-api": {"repos/*/*/releases/**"}}, - Auth: token, - }, - }, - { - name: "credentials for another host", - upstream: UpstreamConfig{ - Generic: githubAPI, - Auth: map[string]AuthConfig{"https://npm.example": {Type: "bearer", Token: "t"}}, - }, - }, - { - name: "paths for an unknown upstream", - upstream: UpstreamConfig{ - Generic: githubAPI, - GenericPaths: map[string][]string{"github": {"*/*/releases/**"}}, - }, - wantErr: true, - }, - { - name: "bad pattern", - upstream: UpstreamConfig{ - Generic: githubAPI, - GenericPaths: map[string][]string{"github-api": {"["}}, - }, - wantErr: true, - }, - { - name: "empty pattern", - upstream: UpstreamConfig{ - Generic: githubAPI, - GenericPaths: map[string][]string{"github-api": {""}}, - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := tt.upstream.Validate(); (err != nil) != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestGenericPathAllowed(t *testing.T) { - patterns := []string{"repos/*/*/releases/**", "rate_limit"} - tests := []struct { - path string - want bool - }{ - {"repos/jqlang/jq/releases/tags/jq-1.7.1", true}, - {"repos/jqlang/jq/releases/latest", true}, - {"rate_limit", true}, - {"repos/jqlang/jq/releases", false}, - {"repos/acme/private/contents/secret.txt", false}, - {"user/repos", false}, - {"repos/jqlang/jq/releases/%2e%2e", false}, - {"repos/jqlang/jq?/releases/x", false}, - {"repos/jqlang/jq/releases/x#y", false}, - {`repos/jqlang/jq/releases/x\y`, false}, - } - for _, tt := range tests { - if got := GenericPathAllowed(patterns, tt.path); got != tt.want { - t.Errorf("GenericPathAllowed(%q) = %v, want %v", tt.path, got, tt.want) - } - } - if !GenericPathAllowed(nil, "user/repos") { - t.Error("GenericPathAllowed without patterns refused a path") - } -} - -func TestLoadGenericPaths(t *testing.T) { - path := filepath.Join(t.TempDir(), "config.yaml") - data := []byte(` -upstream: - generic: - github-api: https://api.github.com - generic_paths: - github-api: - - "repos/*/*/releases/**" - auth: - "https://api.github.com": - type: bearer - token: t -`) - if err := os.WriteFile(path, data, 0o600); err != nil { - t.Fatal(err) - } - cfg, err := Load(path) - if err != nil { - t.Fatalf("Load: %v", err) - } - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate: %v", err) - } - if got := cfg.Upstream.GenericPaths["github-api"]; len(got) != 1 || got[0] != "repos/*/*/releases/**" { - t.Errorf("generic_paths = %v", cfg.Upstream.GenericPaths) - } -} - -func TestAPKSnapshots(t *testing.T) { - cfg := Default() - if got := cfg.ParseAPKSnapshots(); got != 0 { - t.Errorf("default ParseAPKSnapshots = %v, want 0", got) - } - - for value, want := range map[string]time.Duration{ - "90d": 90 * 24 * time.Hour, - "12h": 12 * time.Hour, - "0": 0, - } { - cfg.Upstream.APKSnapshots = value - if err := cfg.Validate(); err != nil { - t.Errorf("Validate(%q): %v", value, err) - } - if got := cfg.ParseAPKSnapshots(); got != want { - t.Errorf("ParseAPKSnapshots(%q) = %v, want %v", value, got, want) - } - } - - for _, value := range []string{"forever", "-1d"} { - cfg.Upstream.APKSnapshots = value - if err := cfg.Validate(); err == nil { - t.Errorf("Validate(%q) succeeded, want an error", value) - } - } - - cfg = Default() - t.Setenv("PROXY_UPSTREAM_APK_SNAPSHOTS", "30d") - cfg.LoadFromEnv() - if cfg.Upstream.APKSnapshots != "30d" { - t.Errorf("APKSnapshots from env = %q, want 30d", cfg.Upstream.APKSnapshots) - } -} - -func TestCooldownShortest(t *testing.T) { - for _, tc := range []struct { - name string - cfg *CooldownConfig - want time.Duration - }{ - {"nil", nil, 0}, - {"none", &CooldownConfig{Default: "0"}, 0}, - {"default", &CooldownConfig{Default: "3d"}, 72 * time.Hour}, - {"ecosystem", &CooldownConfig{Default: "3d", Ecosystems: map[string]string{"npm": "2d", "deb": "0"}}, 48 * time.Hour}, - {"package", &CooldownConfig{Default: "3d", Packages: map[string]string{"pkg:npm/lodash": "6h"}}, 6 * time.Hour}, - {"pattern", &CooldownConfig{Default: "0", PackagePatterns: map[string]string{"pkg:npm/@acme/*": "90m"}}, 90 * time.Minute}, - {"invalid ignored", &CooldownConfig{Default: "3d", Ecosystems: map[string]string{"npm": "soon"}}, 72 * time.Hour}, - } { - if got := tc.cfg.Shortest(); got != tc.want { - t.Errorf("%s: Shortest() = %v, want %v", tc.name, got, tc.want) - } - } -} - -func TestLoadBoolsFromEnvParseStrictly(t *testing.T) { - t.Run("no turns off a setting from the config file", func(t *testing.T) { - t.Setenv("PROXY_TRUST_FORWARDED_FOR", "no") - cfg := Default() - cfg.TrustForwardedFor = true - cfg.LoadFromEnv() - if cfg.TrustForwardedFor { - t.Error("TrustForwardedFor = true, want false") - } - }) - - tests := []struct { - value string - start bool - want bool - warnings int - }{ - {"true", false, true, 0}, - {"True", false, true, 0}, - {" TRUE ", false, true, 0}, - {"1", false, true, 0}, - {"false", true, false, 0}, - {"FALSE", true, false, 0}, - {"0", true, false, 0}, - {"yes", false, true, 0}, - {" On ", false, true, 0}, - {"no", true, false, 0}, - {"OFF", true, false, 0}, - {"maybe", false, false, 1}, - {"ture", true, true, 1}, - } - for _, tt := range tests { - t.Run(tt.value, func(t *testing.T) { - t.Setenv("PROXY_SCANNING_ENABLED", tt.value) - cfg := Default() - cfg.Scanning.Enabled = tt.start - cfg.LoadFromEnv() - if cfg.Scanning.Enabled != tt.want { - t.Errorf("Scanning.Enabled = %v, want %v", cfg.Scanning.Enabled, tt.want) - } - if got := len(cfg.EnvWarnings()); got != tt.warnings { - t.Errorf("len(EnvWarnings()) = %d, want %d: %v", got, tt.warnings, cfg.EnvWarnings()) - } - }) - } -} - -func TestScannerEcosystems(t *testing.T) { - cfg := ScanningConfig{ - Enabled: true, - SigningKey: "s3cret", - Scanners: []ScannerConfig{{ - Name: "clamav", URL: "http://scanner.invalid/scan", - Ecosystems: []string{"go", "Docker", "rubygems", "npm", "arch", "git"}, - }}, - } - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate() error: %v", err) - } - want := []string{"golang", "oci", "gem", "npm", "arch", "git"} - if got := cfg.Scanners[0].Ecosystems; !slices.Equal(got, want) { - t.Errorf("Ecosystems = %v, want %v", got, want) - } - - cfg.Scanners[0].Ecosystems = []string{"npm", "pypy"} - err := cfg.Validate() - if err == nil || !strings.Contains(err.Error(), "pypy") { - t.Errorf("Validate() error = %v, want unknown ecosystem pypy", err) - } -} - -func TestParseSizeRejectsOutOfRange(t *testing.T) { - for _, s := range []string{"-5GB", "-1", "-0.5K", "NaNGB", "InfGB", "+InfB", "1e30GB", "9999999999TB", "0.5B", "0.000000000001GB"} { - if got, err := ParseSize(s); err == nil { - t.Errorf("ParseSize(%q) = %d, want error", s, got) - } - } - cfg := Default() - cfg.Storage.MaxSize = "-5GB" - if err := cfg.Validate(); err == nil { - t.Error("Validate() accepted storage.max_size -5GB") - } -} - -func TestValidateRejectsNegativeTTLs(t *testing.T) { - cfg := Default() - cfg.MetadataTTL = "-5m" - if err := cfg.Validate(); err == nil { - t.Error("Validate() accepted metadata_ttl -5m") - } - - cfg = Default() - cfg.Storage.DirectServeTTL = "-5m" - if err := cfg.Validate(); err == nil { - t.Error("Validate() accepted storage.direct_serve_ttl -5m") - } -} diff --git a/internal/config/cooldown_profiles.go b/internal/config/cooldown_profiles.go deleted file mode 100644 index 225dd20..0000000 --- a/internal/config/cooldown_profiles.go +++ /dev/null @@ -1,64 +0,0 @@ -package config - -import ( - "fmt" - "maps" - "regexp" - "slices" - - "github.com/git-pkgs/cooldown" -) - -var cooldownProfileName = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) - -// Policy builds the policy for this URL's cooldown settings. -func (c *CooldownConfig) Policy() (*cooldown.Policy, error) { - return cooldown.NewPolicy(&cooldown.Config{ - Default: c.Default, Ecosystems: c.Ecosystems, Packages: c.NormalizedPackages(), - }, c.PackagePatterns) -} - -// ValidateProfiles checks names and policies before they become routes. -func (c *CooldownConfig) ValidateProfiles() error { - for _, name := range slices.Sorted(maps.Keys(c.Profiles)) { - profile := c.Profiles[name] - if !cooldownProfileName.MatchString(name) { - return fmt.Errorf("invalid cooldown profile name %q: use lowercase letters, digits, hyphens or underscores", name) - } - if len(profile.Profiles) > 0 { - return fmt.Errorf("cooldown.profiles.%s cannot contain profiles", name) - } - if err := profile.validateProfileDurations(); err != nil { - return fmt.Errorf("cooldown.profiles.%s: %w", name, err) - } - if _, err := profile.Policy(); err != nil { - return fmt.Errorf("cooldown.profiles.%s: %w", name, err) - } - } - return nil -} - -func (c *CooldownConfig) validateProfileDurations() error { - values := map[string]string{"default": c.Default} - for group, overrides := range map[string]map[string]string{ - "ecosystems": c.Ecosystems, "packages": c.Packages, "package_patterns": c.PackagePatterns, - } { - for key, value := range overrides { - values[group+"."+key] = value - } - } - for _, key := range slices.Sorted(maps.Keys(values)) { - value := values[key] - if key == "default" && value == "" { - continue - } - duration, err := cooldown.ParseDuration(value) - if err != nil { - return fmt.Errorf("invalid %s: %w", key, err) - } - if duration < 0 { - return fmt.Errorf("invalid %s: duration must be non-negative", key) - } - } - return nil -} diff --git a/internal/config/cooldown_profiles_test.go b/internal/config/cooldown_profiles_test.go deleted file mode 100644 index 6f10362..0000000 --- a/internal/config/cooldown_profiles_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -func TestLoadCooldownProfiles(t *testing.T) { - filename := filepath.Join(t.TempDir(), "config.yaml") - data := `cooldown: - default: 7d - ecosystems: - npm: "0" - profiles: - 3d: - default: 3d - packages: - "pkg:pypi/Django": "0" - strict: - default: 14d - package_patterns: - "pkg:npm/@example/*": 1d -` - if err := os.WriteFile(filename, []byte(data), 0o600); err != nil { - t.Fatal(err) - } - cfg, err := Load(filename) - if err != nil { - t.Fatal(err) - } - if err := cfg.Validate(); err != nil { - t.Fatal(err) - } - profile := cfg.Cooldown.Profiles["3d"] - policy, err := profile.Policy() - if err != nil { - t.Fatal(err) - } - if got := policy.For("npm", "pkg:npm/demo"); got != 72*time.Hour { - t.Fatalf("profile inherited global ecosystem override: %v", got) - } - if got := policy.For("pypi", "pkg:pypi/django"); got != 0 { - t.Fatalf("canonical package override = %v", got) - } - strict := cfg.Cooldown.Profiles["strict"] - policy, err = strict.Policy() - if err != nil { - t.Fatal(err) - } - if got := policy.For("npm", "pkg:npm/%40example/tool"); got != 24*time.Hour { - t.Fatalf("profile pattern = %v", got) - } -} - -func TestLoadRejectsInvalidCooldownProfiles(t *testing.T) { - for _, tc := range []struct{ name, profile, want string }{ - {"path", `"a/b": {default: 3d}`, "profile name"}, - {"uppercase", `Strict: {default: 3d}`, "profile name"}, - {"empty", `"": {default: 3d}`, "profile name"}, - {"duration", `strict: {default: nope}`, "cooldown.profiles.strict"}, - {"pattern", `strict: {package_patterns: {"pkg:npm/[": 3d}}`, "cooldown.profiles.strict"}, - {"nested", `strict: {profiles: {nested: {default: 3d}}}`, "cannot contain profiles"}, - } { - t.Run(tc.name, func(t *testing.T) { - filename := filepath.Join(t.TempDir(), "config.yaml") - if err := os.WriteFile(filename, []byte("cooldown:\n profiles:\n "+tc.profile+"\n"), 0o600); err != nil { - t.Fatal(err) - } - cfg, err := Load(filename) - if err == nil { - err = cfg.Validate() - } - if err == nil || !strings.Contains(err.Error(), tc.want) { - t.Fatalf("Load error = %v, want %q", err, tc.want) - } - }) - } -} diff --git a/internal/config/denylist.go b/internal/config/denylist.go deleted file mode 100644 index 20c1fca..0000000 --- a/internal/config/denylist.go +++ /dev/null @@ -1,6 +0,0 @@ -package config - -// DenylistConfig identifies exact package versions that must not be served. -type DenylistConfig struct { - Packages []string `json:"packages" yaml:"packages"` -} diff --git a/internal/config/denylist_test.go b/internal/config/denylist_test.go deleted file mode 100644 index f17ad94..0000000 --- a/internal/config/denylist_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - "testing" -) - -func TestDenylistConfig(t *testing.T) { - for _, tc := range []struct{ name, body string }{ - {"yaml", "denylist:\n packages:\n - 'pkg:pypi/requests@2.31.0'\n"}, - {"json", `{"denylist":{"packages":["pkg:pypi/requests@2.31.0"]}}`}, - } { - t.Run(tc.name, func(t *testing.T) { - file := filepath.Join(t.TempDir(), "config."+tc.name) - if err := os.WriteFile(file, []byte(tc.body), 0o600); err != nil { - t.Fatal(err) - } - cfg, err := Load(file) - if err != nil { - t.Fatal(err) - } - if len(cfg.Denylist.Packages) != 1 || cfg.Denylist.Packages[0] != "pkg:pypi/requests@2.31.0" { - t.Fatalf("denylist = %+v", cfg.Denylist) - } - if err := cfg.Validate(); err != nil { - t.Fatal(err) - } - cfg.Denylist.Packages = []string{"pkg:pypi/requests"} - if err := cfg.Validate(); err == nil { - t.Fatal("unversioned denylist entry accepted") - } - }) - } -} diff --git a/internal/config/git.go b/internal/config/git.go deleted file mode 100644 index 860adb7..0000000 --- a/internal/config/git.go +++ /dev/null @@ -1,224 +0,0 @@ -package config - -import ( - "encoding/json" - "fmt" - "path" - "strings" - "time" - - "gopkg.in/yaml.v3" -) - -const ( - defaultGitCacheDir = "./cache/git" - defaultGitMaxRepoSize = "5GB" - defaultGitRefreshWait = "5s" - defaultGitStaleMaxAge = "24h" -) - -// GitUpstream is an upstream.git entry: a Git server's base URL and, -// optionally, the repositories under it that may be served. In YAML or JSON -// it is either the URL alone or an object with url and repos. -type GitUpstream struct { - URL string `json:"url" yaml:"url"` - - // Repos lists path.Match patterns for repository paths under URL, such - // as "mediawiki/*". A pattern ending in "/**" also matches every - // repository nested below its prefix: "mediawiki/**" matches - // mediawiki/skins/Vector. Empty allows every repository. Required when - // upstream.auth has credentials for the URL, since everyone who can - // reach the proxy can read what it serves. - Repos []string `json:"repos" yaml:"repos"` -} - -// UnmarshalYAML accepts a bare URL as well as the object form. -func (g *GitUpstream) UnmarshalYAML(node *yaml.Node) error { - if node.Kind == yaml.ScalarNode { - return node.Decode(&g.URL) - } - type plain GitUpstream - return node.Decode((*plain)(g)) -} - -// UnmarshalJSON accepts a bare URL as well as the object form. -func (g *GitUpstream) UnmarshalJSON(data []byte) error { - if err := json.Unmarshal(data, &g.URL); err == nil { - return nil - } - type plain GitUpstream - return json.Unmarshal(data, (*plain)(g)) -} - -// AllowsRepo reports whether repo, a path under URL, may be served. -func (g GitUpstream) AllowsRepo(repo string) bool { - if len(g.Repos) == 0 { - return true - } - for _, pattern := range g.Repos { - if matchRepoPattern(pattern, repo) { - return true - } - } - return false -} - -func matchRepoPattern(pattern, repo string) bool { - prefix, nested := strings.CutSuffix(pattern, "/**") - if !nested { - ok, _ := path.Match(pattern, repo) - return ok - } - segments := strings.Count(prefix, "/") + 1 - parts := strings.SplitN(repo, "/", segments+1) - if len(parts) <= segments { - return false - } - ok, _ := path.Match(prefix, strings.Join(parts[:segments], "/")) - return ok -} - -// GitConfig configures the Git repository cache behind /git/{name}/. -type GitConfig struct { - // CacheDir holds the cached bare repositories. It must be local disk - // used by this process only, which locks it. Default: ./cache/git. - CacheDir string `json:"cache_dir" yaml:"cache_dir"` - - // MaxSize bounds the cache, counting fetches in progress; the least - // recently used repositories not in use are evicted to stay within it. - // Empty or "0" disables the limit. - MaxSize string `json:"max_size" yaml:"max_size"` - - // MaxRepoSize is the most one repository may take; a clone or fetch that - // would pass it fails. "0" disables the limit. Default: 5GB. MaxSize - // caps it when set. - MaxRepoSize string `json:"max_repo_size" yaml:"max_repo_size"` - - // RefreshWait is how long a request waits for the upstream's branches - // and tags to be checked before it is served the cached copy. A fetch of - // new commits the check finds is waited for in full. Default: 5s. - RefreshWait string `json:"refresh_wait" yaml:"refresh_wait"` - - // StaleMaxAge is how long after the last successful upstream check a - // cached copy may still be served when the upstream fails or is slow. - // Default: 24h. - StaleMaxAge string `json:"stale_max_age" yaml:"stale_max_age"` - - maxSize int64 - maxRepoSize int64 - refreshWait time.Duration - staleMaxAge time.Duration -} - -// MaxSizeBytes returns the parsed MaxSize, 0 for no limit. Validate must run first. -func (g *GitConfig) MaxSizeBytes() int64 { return g.maxSize } - -// MaxRepoSizeBytes returns the parsed MaxRepoSize, 0 for no limit. Validate must run first. -func (g *GitConfig) MaxRepoSizeBytes() int64 { return g.maxRepoSize } - -// RefreshWaitDuration returns the parsed RefreshWait. Validate must run first. -func (g *GitConfig) RefreshWaitDuration() time.Duration { return g.refreshWait } - -// StaleMaxAgeDuration returns the parsed StaleMaxAge. Validate must run first. -func (g *GitConfig) StaleMaxAgeDuration() time.Duration { return g.staleMaxAge } - -// Validate checks the Git cache settings, applying defaults where unset. -func (g *GitConfig) Validate() error { - if g.CacheDir == "" { - g.CacheDir = defaultGitCacheDir - } - if g.MaxRepoSize == "" { - g.MaxRepoSize = defaultGitMaxRepoSize - } - if g.RefreshWait == "" { - g.RefreshWait = defaultGitRefreshWait - } - if g.StaleMaxAge == "" { - g.StaleMaxAge = defaultGitStaleMaxAge - } - - var err error - if g.maxSize, err = parseNonNegativeSize("git.max_size", g.MaxSize); err != nil { - return err - } - if g.maxRepoSize, err = parseNonNegativeSize("git.max_repo_size", g.MaxRepoSize); err != nil { - return err - } - if g.refreshWait, err = parseNonNegativeDuration("git.refresh_wait", g.RefreshWait); err != nil { - return err - } - if g.staleMaxAge, err = parseNonNegativeDuration("git.stale_max_age", g.StaleMaxAge); err != nil { - return err - } - return nil -} - -func parseNonNegativeSize(field, value string) (int64, error) { - size, err := ParseSize(value) - if err != nil { - return 0, fmt.Errorf("invalid %s: %w", field, err) - } - if size < 0 { - return 0, fmt.Errorf("invalid %s %q: must not be negative", field, value) - } - return size, nil -} - -func parseNonNegativeDuration(field, value string) (time.Duration, error) { - d, err := time.ParseDuration(value) - if err != nil { - return 0, fmt.Errorf("invalid %s %q: %w", field, value, err) - } - if d < 0 { - return 0, fmt.Errorf("invalid %s %q: must not be negative", field, value) - } - return d, nil -} - -// validateGitUpstreams checks upstream.git names, URLs and repository -// patterns. An upstream that upstream.auth has credentials for must list its -// repositories, so configuring a token does not open every repository it can -// read to everyone who can reach the proxy. -func (u *UpstreamConfig) validateGitUpstreams() error { - for name, upstream := range u.Git { - field := "upstream.git." + name - if name == "" || name == "." || name == ".." || name[0] == '.' || strings.ContainsAny(name, `/\\`) { - return fmt.Errorf("invalid upstream.git name %q", name) - } - if err := validateAbsoluteURL(field, upstream.URL); err != nil { - return err - } - if scheme, _, _ := strings.Cut(upstream.URL, "://"); !strings.EqualFold(scheme, "https") && !strings.EqualFold(scheme, "http") { - return fmt.Errorf("invalid %s %q: only http and https are supported", field, upstream.URL) - } - for _, pattern := range upstream.Repos { - if _, err := path.Match(strings.TrimSuffix(pattern, "/**"), ""); pattern == "" || err != nil { - return fmt.Errorf("invalid %s.repos pattern %q", field, pattern) - } - } - if len(upstream.Repos) == 0 && u.HasAuthWithin(upstream.URL) { - return fmt.Errorf("%s needs repos: upstream.auth has credentials for %s, "+ - "and everyone who can reach the proxy could read every repository they allow", field, upstream.URL) - } - } - return nil -} - -// HasAuthWithin reports whether an upstream.auth scope covers baseURL or a -// path under it. -func (u *UpstreamConfig) HasAuthWithin(baseURL string) bool { - base, err := parseAuthURL(baseURL) - if err != nil { - return false - } - for scope := range u.Auth { - configured, err := parseAuthURL(scope) - if err != nil { - continue - } - if authURLMatches(configured, base) || authURLMatches(base, configured) { - return true - } - } - return false -} diff --git a/internal/config/git_test.go b/internal/config/git_test.go deleted file mode 100644 index 6c9b47b..0000000 --- a/internal/config/git_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package config - -import ( - "encoding/json" - "os" - "path/filepath" - "slices" - "testing" - "time" - - "gopkg.in/yaml.v3" -) - -func TestGitUpstreamAcceptsURLOrObject(t *testing.T) { - want := map[string]GitUpstream{ - "github": {URL: "https://github.com"}, - "wikimedia": {URL: "https://gitlab.wikimedia.org", Repos: []string{"mediawiki/*"}}, - } - - var fromYAML struct { - Git map[string]GitUpstream `yaml:"git"` - } - err := yaml.Unmarshal([]byte(` -git: - github: "https://github.com" - wikimedia: - url: "https://gitlab.wikimedia.org" - repos: ["mediawiki/*"] -`), &fromYAML) - if err != nil { - t.Fatalf("YAML: %v", err) - } - - var fromJSON struct { - Git map[string]GitUpstream `json:"git"` - } - err = json.Unmarshal([]byte(`{"git": { - "github": "https://github.com", - "wikimedia": {"url": "https://gitlab.wikimedia.org", "repos": ["mediawiki/*"]} - }}`), &fromJSON) - if err != nil { - t.Fatalf("JSON: %v", err) - } - - for format, got := range map[string]map[string]GitUpstream{"YAML": fromYAML.Git, "JSON": fromJSON.Git} { - for name, w := range want { - if g := got[name]; g.URL != w.URL || !slices.Equal(g.Repos, w.Repos) { - t.Errorf("%s %s = %+v, want %+v", format, name, g, w) - } - } - } -} - -func TestGitUpstreamAllowsRepo(t *testing.T) { - open := GitUpstream{URL: "https://github.com"} - if !open.AllowsRepo("any/repo") { - t.Error("an upstream without repos refused a repository") - } - listed := GitUpstream{URL: "https://gitlab.example", Repos: []string{"mediawiki/*", "tools/one", "group/**"}} - for repo, want := range map[string]bool{ - "mediawiki/core": true, - "tools/one": true, - "tools/two": false, - "mediawiki/extensions/Echo": false, - "group/sub/project": true, - "group/project": true, - "group": false, - "groups/project": false, - } { - if got := listed.AllowsRepo(repo); got != want { - t.Errorf("AllowsRepo(%q) = %v, want %v", repo, got, want) - } - } -} - -func TestValidateGitUpstreams(t *testing.T) { - tests := []struct { - name string - upstream UpstreamConfig - wantErr bool - }{ - {name: "public", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "https://github.com"}}}}, - {name: "relative URL", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "github.com"}}}, wantErr: true}, - {name: "name with slash", upstream: UpstreamConfig{Git: map[string]GitUpstream{"a/b": {URL: "https://github.com"}}}, wantErr: true}, - {name: "name with leading dot", upstream: UpstreamConfig{Git: map[string]GitUpstream{".tmp": {URL: "https://github.com"}}}, wantErr: true}, - {name: "bad pattern", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "https://github.com", Repos: []string{"["}}}}, wantErr: true}, - {name: "nested pattern", upstream: UpstreamConfig{Git: map[string]GitUpstream{"gerrit": {URL: "https://gerrit.example/r", Repos: []string{"mediawiki/**"}}}}}, - {name: "ssh URL", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "ssh://git@github.com"}}}, wantErr: true}, - { - name: "credentials without repos", - upstream: UpstreamConfig{ - Git: map[string]GitUpstream{"private": {URL: "https://git.example"}}, - Auth: map[string]AuthConfig{"https://git.example": {Type: "bearer", Token: "t"}}, - }, - wantErr: true, - }, - { - name: "credentials for a path under the upstream", - upstream: UpstreamConfig{ - Git: map[string]GitUpstream{"private": {URL: "https://git.example"}}, - Auth: map[string]AuthConfig{"https://git.example/team": {Type: "bearer", Token: "t"}}, - }, - wantErr: true, - }, - { - name: "credentials with repos", - upstream: UpstreamConfig{ - Git: map[string]GitUpstream{"private": {URL: "https://git.example", Repos: []string{"team/*"}}}, - Auth: map[string]AuthConfig{"https://git.example": {Type: "bearer", Token: "t"}}, - }, - }, - { - name: "credentials for another host", - upstream: UpstreamConfig{ - Git: map[string]GitUpstream{"github": {URL: "https://github.com"}}, - Auth: map[string]AuthConfig{"https://npm.example": {Type: "bearer", Token: "t"}}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := tt.upstream.Validate(); (err != nil) != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestGitConfigDefaults(t *testing.T) { - var g GitConfig - if err := g.Validate(); err != nil { - t.Fatalf("Validate: %v", err) - } - if g.CacheDir != defaultGitCacheDir || g.MaxSizeBytes() != 0 || g.MaxRepoSizeBytes() != 5<<30 || - g.RefreshWaitDuration() != 5*time.Second || g.StaleMaxAgeDuration() != 24*time.Hour { - t.Errorf("defaults = %+v", g) - } - - for _, bad := range []GitConfig{{MaxSize: "lots"}, {RefreshWait: "soon"}, {StaleMaxAge: "-1h"}} { - if err := bad.Validate(); err == nil { - t.Errorf("Validate(%+v) accepted it", bad) - } - } -} - -func TestLoadGitConfig(t *testing.T) { - path := filepath.Join(t.TempDir(), "config.yaml") - data := []byte(` -upstream: - git: - github: https://github.com -git: - cache_dir: /data/git - max_size: 50GB -`) - if err := os.WriteFile(path, data, 0o600); err != nil { - t.Fatal(err) - } - cfg, err := Load(path) - if err != nil { - t.Fatalf("Load: %v", err) - } - t.Setenv("PROXY_GIT_REFRESH_WAIT", "2s") - cfg.LoadFromEnv() - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate: %v", err) - } - if cfg.Upstream.Git["github"].URL != "https://github.com" || cfg.Git.CacheDir != "/data/git" || - cfg.Git.MaxSizeBytes() != 50<<30 || cfg.Git.RefreshWaitDuration() != 2*time.Second { - t.Errorf("loaded %+v and %+v", cfg.Upstream.Git, cfg.Git) - } -} diff --git a/internal/config/hosted.go b/internal/config/hosted.go deleted file mode 100644 index 9b67e28..0000000 --- a/internal/config/hosted.go +++ /dev/null @@ -1,77 +0,0 @@ -package config - -import ( - "fmt" - "strings" -) - -const defaultHostedMaxUploadSize = "100MB" - -// HostedConfig configures packages pushed to the proxy through the -// /api/hosted endpoints. Pushed versions are served in place of upstream -// versions with the same number, so writes always require a token. -type HostedConfig struct { - // Enabled registers the /api/hosted endpoints. Off by default. - Enabled bool `json:"enabled" yaml:"enabled"` - - // Token is the bearer token required to push and delete packages. - // Required when Enabled. Supports ${VAR_NAME} expansion like AuthConfig - // fields. - Token string `json:"token" yaml:"token"` - - // MaxUploadSize caps a single pushed archive (e.g., "100MB"). Must be > 0. - // Default: "100MB". - MaxUploadSize string `json:"max_upload_size" yaml:"max_upload_size"` - - // NPMScopes are npm scopes (e.g. "@acme") served only from pushed - // packages. The upstream registry is never asked about them, so a package - // published there under the same name can't be mixed in. - NPMScopes []string `json:"npm_scopes" yaml:"npm_scopes"` - - // ComposerVendors are Composer vendors (e.g. "acme") served only from - // pushed packages, like NPMScopes. - ComposerVendors []string `json:"composer_vendors" yaml:"composer_vendors"` - - maxUploadBytes int64 -} - -// TokenExpanded returns Token with ${VAR_NAME} references expanded. -func (h *HostedConfig) TokenExpanded() string { - return expandEnv(h.Token) -} - -// MaxUploadBytes returns the parsed MaxUploadSize. Validate must run first. -func (h *HostedConfig) MaxUploadBytes() int64 { - return h.maxUploadBytes -} - -// Validate checks the hosted package settings, applying the default upload -// size if unset. -func (h *HostedConfig) Validate() error { - if h.MaxUploadSize == "" { - h.MaxUploadSize = defaultHostedMaxUploadSize - } - size, err := ParseSize(h.MaxUploadSize) - if err != nil { - return fmt.Errorf("invalid hosted.max_upload_size: %w", err) - } - if size <= 0 { - return fmt.Errorf("invalid hosted.max_upload_size %q: must be > 0", h.MaxUploadSize) - } - h.maxUploadBytes = size - - if h.Enabled && h.TokenExpanded() == "" { - return fmt.Errorf("hosted.token is required when hosted.enabled is true") - } - for _, scope := range h.NPMScopes { - if len(scope) < 2 || scope[0] != '@' || strings.Contains(scope, "/") { - return fmt.Errorf("invalid hosted.npm_scopes entry %q: must be a scope such as \"@acme\"", scope) - } - } - for _, vendor := range h.ComposerVendors { - if vendor == "" || strings.Contains(vendor, "/") { - return fmt.Errorf("invalid hosted.composer_vendors entry %q: must be a vendor such as \"acme\"", vendor) - } - } - return nil -} diff --git a/internal/config/hosted_test.go b/internal/config/hosted_test.go deleted file mode 100644 index e7ad451..0000000 --- a/internal/config/hosted_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package config - -import ( - "slices" - "testing" -) - -func TestHostedConfigValidate(t *testing.T) { - t.Setenv("HOSTED_TEST_TOKEN", "from-env") - - tests := []struct { - name string - hosted HostedConfig - wantErr bool - }{ - {name: "disabled without token", hosted: HostedConfig{}}, - {name: "enabled without token", hosted: HostedConfig{Enabled: true}, wantErr: true}, - {name: "enabled with token", hosted: HostedConfig{Enabled: true, Token: "s3cret"}}, - {name: "token from env reference", hosted: HostedConfig{Enabled: true, Token: "${HOSTED_TEST_TOKEN}"}}, - {name: "token env reference unset", hosted: HostedConfig{Enabled: true, Token: "${HOSTED_TEST_UNSET}"}, wantErr: true}, - {name: "invalid upload size", hosted: HostedConfig{Enabled: true, Token: "s3cret", MaxUploadSize: "lots"}, wantErr: true}, - {name: "zero upload size", hosted: HostedConfig{Enabled: true, Token: "s3cret", MaxUploadSize: "0"}, wantErr: true}, - {name: "npm scopes", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"@acme", "@acme-internal"}}}, - {name: "npm scope without @", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"acme"}}, wantErr: true}, - {name: "npm scope with package", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"@acme/private"}}, wantErr: true}, - {name: "bare @", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"@"}}, wantErr: true}, - {name: "composer vendors", hosted: HostedConfig{Enabled: true, Token: "s3cret", ComposerVendors: []string{"acme", "acme-internal"}}}, - {name: "composer vendor with package", hosted: HostedConfig{Enabled: true, Token: "s3cret", ComposerVendors: []string{"acme/private"}}, wantErr: true}, - {name: "empty composer vendor", hosted: HostedConfig{Enabled: true, Token: "s3cret", ComposerVendors: []string{""}}, wantErr: true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := tt.hosted.Validate() - if (err != nil) != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestHostedConfigDefaults(t *testing.T) { - h := HostedConfig{Enabled: true, Token: "${HOSTED_TEST_TOKEN}"} - t.Setenv("HOSTED_TEST_TOKEN", "from-env") - if err := h.Validate(); err != nil { - t.Fatalf("Validate: %v", err) - } - if got := h.TokenExpanded(); got != "from-env" { - t.Errorf("TokenExpanded() = %q, want from-env", got) - } - if got := h.MaxUploadBytes(); got != 100<<20 { - t.Errorf("MaxUploadBytes() = %d, want %d", got, 100<<20) - } - - h.MaxUploadSize = "2GB" - if err := h.Validate(); err != nil { - t.Fatalf("Validate: %v", err) - } - if got := h.MaxUploadBytes(); got != 2<<30 { - t.Errorf("MaxUploadBytes() = %d, want %d", got, int64(2)<<30) - } -} - -func TestValidateRejectsHostedWithoutCachedArtifacts(t *testing.T) { - cfg := Default() - cfg.Hosted = HostedConfig{Enabled: true, Token: "s3cret"} - cfg.Storage.CacheArtifacts = false - if err := cfg.Validate(); err == nil { - t.Error("Validate() accepted hosted.enabled with storage.cache_artifacts: false") - } -} - -func TestLoadFromEnvHosted(t *testing.T) { - cfg := Default() - t.Setenv("PROXY_HOSTED_ENABLED", "true") - t.Setenv("PROXY_HOSTED_TOKEN", "env-token") - t.Setenv("PROXY_HOSTED_MAX_UPLOAD_SIZE", "50MB") - t.Setenv("PROXY_HOSTED_NPM_SCOPES", "@acme, @acme-internal") - t.Setenv("PROXY_HOSTED_COMPOSER_VENDORS", "acme") - - cfg.LoadFromEnv() - - if !cfg.Hosted.Enabled { - t.Error("Hosted.Enabled not loaded from PROXY_HOSTED_ENABLED") - } - if cfg.Hosted.Token != "env-token" { - t.Errorf("Hosted.Token = %q, want env-token", cfg.Hosted.Token) - } - if cfg.Hosted.MaxUploadSize != "50MB" { - t.Errorf("Hosted.MaxUploadSize = %q, want 50MB", cfg.Hosted.MaxUploadSize) - } - if !slices.Equal(cfg.Hosted.NPMScopes, []string{"@acme", "@acme-internal"}) { - t.Errorf("Hosted.NPMScopes = %q", cfg.Hosted.NPMScopes) - } - if !slices.Equal(cfg.Hosted.ComposerVendors, []string{"acme"}) { - t.Errorf("Hosted.ComposerVendors = %q", cfg.Hosted.ComposerVendors) - } -} diff --git a/internal/config/maven_group_roots.txt b/internal/config/maven_group_roots.txt deleted file mode 100644 index b199222..0000000 --- a/internal/config/maven_group_roots.txt +++ /dev/null @@ -1,1228 +0,0 @@ -# First path segments of group IDs on Maven Central and the Gradle Plugin -# Portal, which the default repository falls back to, plus roots from -# Google's Maven repository. A maven_repositories name from this list would -# hide those groups in the default repository. Regenerate with: -# for u in https://repo1.maven.org/maven2/ https://plugins.gradle.org/m2/; do -# curl -s $u | grep -o 'href="[^"]*/"' | sed 's/href="//;s/\/"//' | grep -v '^\.\.$' -# done | sort -u # then add android and androidx -0x -abbot -ac -academy -acegisecurity -activation -activecluster -activeio -activemq -activemq-jaxb -activesoap -activespace -ad -adarwin -ae -aelfred -aero -africa -ag -agency -agorapulse -ai -aislib -ajk -al -all -alm -altrmi -am -amdonov -amnl -android -androidx -andromda -annogen -ant -ant-contrib -ant-doxygen -ant4eclipse -antlr -anttex -anverus -anvil -aopalliance -apache-jaxme -apl -app -aptconvert -ar -argelbargel -args4j -art -as -ashkay -ashkelon -asia -asm -aspectj -aspectwerkz -at -au -audio -avalon -avalon-activation -avalon-apps -avalon-composition -avalon-cornerstone -avalon-extension -avalon-framework -avalon-http -avalon-logging -avalon-logkit -avalon-meta -avalon-phoenix -avalon-repository -avalon-util -aws -ax -axion -axis -axis2 -az -azote -ba -backport-util-concurrent -backport175 -bairesqa -band -barsuift -baseline -batik -bayern -bcel -bd -be -beehive -beer -berkano -berkeleydb -berlin -best -bet -bg -bi -bike -binky -bio -biz -black -blissed -blog -blue -boo -bot -bouncycastle -box -boxstuff -br -brownshome -bsf -bsh -bue -build -builders -bump -burlap -buzz -by -bz -c10n -c3p0 -ca -cab -cactus -cafe -calces -cam -camp -capital -cards -care -cargo -cas -casa -cash -castor -cat -cc -ccd-sdk -cd -center -cern -cewolf -cf -cglib -cgoit -ch -charlotte -chat -cheap -checkstyle -church -ci -city -cj -cl -classworlds -click -clickstream -clinic -clirr -cloud -clover -club -cm -cmsdeploy -cn -co -coach -cobertura -coconut -cocoon -code -code316 -codehaus -CodeLineCounter -codes -coffee -colaba -colt -com -commons-attributes -commons-beanutils -commons-betwixt -commons-chain -commons-cli -commons-codec -commons-collections -commons-compress -commons-configuration -commons-daemon -commons-dbcp -commons-dbutils -commons-digester -commons-discovery -commons-el -commons-email -commons-fileupload -commons-grafolia -commons-grant -commons-graph -commons-http -commons-httpclient -commons-i18n -commons-io -commons-jdbc2pool -commons-jelly -commons-jexl -commons-jux -commons-jxpath -commons-lang -commons-latka -commons-launcher -commons-logging -commons-math -commons-messenger -commons-modeler -commons-naming -commons-net -commons-pool -commons-primitives -commons-resources -commons-scxml -commons-services -commons-sql -commons-test -commons-threadpool -commons-transaction -commons-util -commons-validator -commons-vfs -commons-xo -community -company -computer -concurrent -consulting -continuum -controlhaus -cool -coop -cornerstone-connection -cornerstone-datasources -cornerstone-scheduler -cornerstone-sockets -cornerstone-store -cornerstone-threads -cos -cover -craft -crimson -cryptix -cssparser -cup -cv -cwoongc -cx -cz -d-haven-event -d-haven-eventbus -d-haven-mpool -d3code -daemoniclabs -dalma -dalms -damagecontrol -dataforge -datasift -date -daveight -davethe8 -day -dbunit -de -decaf -decorutils -default -dentaku -dependency -dependency-maven-plugin -design -dev -digit -digital -dinistiq -directory -directory-asn1 -directory-authx -directory-clients -directory-naming -directory-network -directory-protocols -directory-shared -displaytag -ditchnet -dk -dm -dna -dnsjava -do -docbook -doccheck -docker -dom4j -domify -doxia -drone -drools -drools-examples -ds2 -dsh-vocabulary -dtdparser -dumbster -dwr -dynaop -earth -easyconf -easymock -ec -echo -echo3 -echo3extras -eclipse -eco -ecs -edenlib -edtftp -edu -education -ee -egor-n -ehcache -ejb -el-impl -elect86 -email -emberio -emetriq -emma -energy -engineer -engineering -enterprise -enterprises -ervacon -es -esdk -esper -eu -eus -events -excalibur -excalibur-altrmi -excalibur-cli -excalibur-collections -excalibur-component -excalibur-component-examples -excalibur-component-tests -excalibur-concurrent -excalibur-configuration -excalibur-containerkit -excalibur-datasource -excalibur-event -excalibur-extension -excalibur-fortress -excalibur-i18n -excalibur-instrument -excalibur-instrument-manager -excalibur-instrument-manager-interfaces -excalibur-io -excalibur-lifecycle -excalibur-logger -excalibur-monitor -excalibur-mpool -excalibur-naming -excalibur-pool -excalibur-sourceresolve -excalibur-store -excalibur-thread -excalibur-util -excalibur-xmlutil -exchange -exist -exml -exo -exolabcore -expert -express -exteca -external -ezy -family -fan -fans -farm -fashion -fastutil -fesi -fi -film -finance -financial -findbugs -firebase -firedancer -fish -fit -flavor -flox -flux -fm -fo -foo -fop -forehead -formproc -foundation -foxtrot -fr -framework -freebxml -freemarker -frl -fulcrum -fun -fwrap -fyi -ga -gabriel -gae -games -garden -gay -gbean -gdn -ge -generama -genjar -genjava -gent -geronimo -geronimo-spec -getui -gg -github -glass -glassfish -gliby -global -gmbh -gnu -gnu-regexp -gov -gq -gr -gradle -gradle-autoconf -gradle-clojure -gradle-jenkins-job-dsl-plugin -graphics -graphlayout -green -greendao -gregdev -grizzly -grizzly-cachetest -groovy -groovy-xmlrpc -groovycalamari -groovyx -group -gryphon -gs -gsbase -guru -haibison -hamburg -health -help -hessian -hibernate -hihex -hivemind -hk -hm -hmcts -hnct -Hoop -host -house -howl -hr -hsqldb -htmlunit -HTTPClient -httpcomponents-httpcore -httpunit -hu -hudson -huqi -i -ical4j -icu -icu4j -id -idb -ie -iirekm -ikakara -il -im -immo -in -inc -industries -info -informa -ink -innig -int -io -iq -ir -is -isorelax -it -itext -ivory -izpack -jaas -jaci -jackcess -jackson -jacl -jaf -jaimbot -jakarta -jakarta-regexp -jalopy -james -janino -jardiff -jarjar -jarsync -jasper-jsr199 -jasperreports -java_cup -java2html -java3d -javacc -javaconfig -javadb -javadoc -javaee -javagroups -javainetlocator -javamail -javamuc -javancss -javanettasks -javassist -javatar -javax -javazoom -javolution -jawin -jaxb -jaxen -jaxme -jaxr-ra -jblanket -jboss -jca -jcache -jcharts -jcifs -jcollardbovy -jcom -jcommon -jcoverage -jcs -jcs-javagroups -jcvsii -jdbc -jdbm -jdepend -jdiff -jdo -jdom -jdring -jdsl -je -jen -jencks -jep -jepi -jersey -jetty -jexcelapi -jface -jfree -jfreechart -jgen -jgoodies -jgraph -jgrapht -jgroups -jhunlang -jimmy -jini -jintention -jisp -jivesoftware -jlibdiff -jline -jmagick -jmaki -jmdns -jmfayard -jmimemagic -jmml -jmock -jms -jmscts -jmsn -joda-time -john-test -jotm -jp -jparsec -jpl -jpox -jpox-dbcp -jpox-enhancer -jpox-java5 -jrexx -jrms -jrobin -jruby -jsch -jsf-extensions -jspapi -jsptags -jstl -jstyle -jta -jtds -jtidy -juddi -judy -jug -jung -junit -junit-addons -junit-doclet -junitperf -juno -JuspayAssets -jwebunit -jxta -jython -k8s -kante -kawa -ke -keith -kg -kh -kim -king -kirin -kiwi -kkdt -kliche -koeln -kohsuke -kotlin2ts -kr -krd -kred -kruse -kt-logging -ktorm-tools -kxml -kxml2 -ky -kz -la -land -lat -lazy -lc -ldapd-common -ldapsdk -lenala -lgbt -li -libgdx-oboe -life -lingo -link -live -lk -llc -locc -log4j -log4unit -loggersoft -logkit -lol -loom -love -lpg -lt -ltd -lu -lucene -luisrjaeger -lv -ly -ma -magicGball -maps -market -markets -marmalade -math -maven -maven-integration-test-helper -maven-javanet-plugin -maven-new -maven-plugins -maven-proxy -maven-taglib -maven-torque-plugin -maven-validator -maven-xdoclet2-plugin -maxq -mckoi -md -me -media -merlin -merlin-developer -merlin-tutorial -messenger -metaclass -mevenide -micro -microcontainer -middlegen -mil -milyn -mk -ml -mm -mm-mysql -mn -mnm -mobi -mockcreator -mockit -mockmaker -mockobjects -mockrunner -modello -moe -monetdb -money -monster -mp -mpern -mrj -ms -mstor -msv -mt -mu -mule -muse -mx -mx4j -my -myfaces -mysql -mz -name -nanning -nanocontainer -nebula -nekohtml -neo -net -netbeans -network -neva -new -nexus -nf -ng -ninja -nl -no -norbert -nova -np -nrlssc -nsuml -nu -null -nz -oauth -odmg -oe -ognl -ojb -ojdbc -old -one -oness -onl -online -open -open-esb -opencypher -openejb -openim -openjms -opennms -opensymphony -openxp -oracle -org -org.elasticsearch -org.elasticsearch.client -org.elasticsearch.distribution.integ-test-zip -org.elasticsearch.distribution.zip -org.elasticsearch.gradle -org.elasticsearch.plugin -org.elasticsearch.test -oro -oscube -ovh -p2psockets -p6spy -page -partners -party -pathwaycommons -patterntesting -payload -pcj -pdfbox -pe -penguin -pet -petridish -ph -phd -photos -piccolo -picocontainer -picounit -pink -piotrmadry -pircbot -pitt -pk -pl -plexus -plj -plugin -plus -pluto-container -pm -pmd -pnuts -poi -poolman -portlet-api -postgresql -prevayler -pro -proctor -productions -profiler -progress -projektor -protop -proxool -proxytoys -pt -pub -pubscribe -pull-parser -pw -py -qa -qdox -qfork -quartz -quest -quilt -radeox -re -readline -red -redhill -redis -redmine -regexp -relaxngDatatype -ren -report -reportrunner -rest -rgoldberg -rhino -ricosys -rip -rkennel -ro -robo-guice -roboguice -rocks -rodeo -roller -rome -rpost -rs -rss4j -rsslibj -ru -rubygems -run -sa -sablecc -sagex -sale -sap -sax -saxon -saxpath -sbnarra -sbs -sc -scabbard -schibsted -schoenja -school -schwarz -science -scot -scout -scraping-engine -se -sebersole -security -securityfilter -seek -serg -servicemix -servicemix-ws -services -servletapi -servlets -setpoint -sfx4j -sg -sh -shellix -shocks -shop -show -si -si6 -sillyexceptions -simple -simple-jms -simple-jndi -site -sk -skaringa -ski -skinlf -slide -smartrics -sn -so -soap -social -software -soimp -solar -solarisrealm -solutions -source -space -speexx -spice -spring -springframework -springmodules -sqlline -srl -sshtools -sslext -st -stapler -statcvs -stax -stax-utils -store -stratum -stream -struts -struts-menu -strutstestcase -studio -stxx -su -subpersistence -subshell -substance -suiterunner -surefire -swanysimon -swarmcache -swehacker -swiss -swt -synapticloop -systems -sysunit -tablelayout -tagalog -tagishauth -taglibrarydoc -taglibs -tagsoup -tambora -tane -tangram -tanukisoft -tanvd -tapestry -tclib -team -tec -tech -technology -tekgenesis -tel -temportalist -textarea -tf -th -thad -thaiopensource -thefireplace -tiffrenderer -tjdo -tk -tl -tmporb -to -today -tokyo -tomcat -tomcat-util -tonic -tools -top -toplink -torque -torque-gen -touchstone -town -toys -tqrg -tr -trade -traer -trail-taglib -tranql -travel -trove -trust -tui -turbine -tv -tw -tylerthrailkill -tyrex -tz -ua -uaihebert -ubique -ud -ug -uispec4j -uk -umple -uno -uno-jar -urbanophile -urlrewrite -us -utry -uy -uz -vast -vc -vdoclet -velocity -velocity-anakia -velocity-dvsl -velocity-tools -ventures -vet -vicJuspay -video -view_inspector -village -vin -vinccool96 -vip -vision -vn -vogle -vontikov -vu -wadi -waena -wang -web -webmacro -website -webtest -weixinkeji -werken-xpath -werkflow -werkz -westhawk -wf -wg -which -wicket -wiki -win -wind57 -woodstox -work -works -world -wrapper -ws -ws-commons -ws-commons-java5 -ws-commons-util -ws-scout -wsdl4j -wsrf -wss4j -wstx -wtf -wurfl -wutka -xajile -xalan -xbean -xdoclet -xdoclet-plugins -xerces -xercesjarv -xfire -xfire-root -xin -xjavadoc -xml-apis -xml-resolver -xml-security -xmlbeans -xmldb -xmlenc -xmlmind -xmlpull -xmlrpc -xmlrpc-helma -xmlunit -xmlwise -xmlwriter -xom -xpp3 -xsddoc -xsdlib -xstream -xtc -xtiff-jai -xxl -xyz -yakworks -yan -yanjun -ymsg -yom -za -zip -zm -zone -zw diff --git a/internal/config/mirror.go b/internal/config/mirror.go deleted file mode 100644 index 7f6c668..0000000 --- a/internal/config/mirror.go +++ /dev/null @@ -1,108 +0,0 @@ -package config - -import ( - "fmt" - "strings" - "time" - - "github.com/git-pkgs/purl" -) - -const minMirrorScheduleInterval = time.Minute - -// MirrorConfig configures mirroring that the server runs on its own. -type MirrorConfig struct { - // Schedules re-mirror container image tags at a fixed interval, so a - // moved tag is picked up even when nobody pulls it. - Schedules []MirrorSchedule `json:"schedules" yaml:"schedules"` -} - -// MirrorSchedule is a set of image PURLs mirrored every Interval. -type MirrorSchedule struct { - // Images are pkg:docker or pkg:oci PURLs with a tag. - Images []string `json:"images" yaml:"images"` - - // Interval is how often the images are mirrored, at least one minute - // (e.g. "6h"). - Interval string `json:"interval" yaml:"interval"` - - // Platforms limit which platforms of an image index are mirrored, as - // os/architecture[/variant]. Empty mirrors every platform. - Platforms []string `json:"platforms" yaml:"platforms"` - - interval time.Duration -} - -// IntervalDuration returns the parsed Interval. Validate must run first. -func (s *MirrorSchedule) IntervalDuration() time.Duration { - return s.interval -} - -// Validate checks every schedule. -func (m *MirrorConfig) Validate() error { - for i := range m.Schedules { - if err := m.Schedules[i].validate(fmt.Sprintf("mirror.schedules[%d]", i)); err != nil { - return err - } - } - return nil -} - -func (s *MirrorSchedule) validate(field string) error { - d, err := time.ParseDuration(s.Interval) - if err != nil { - return fmt.Errorf("invalid %s.interval %q: %w", field, s.Interval, err) - } - if d < minMirrorScheduleInterval { - return fmt.Errorf("invalid %s.interval %q: must be at least %s", field, s.Interval, minMirrorScheduleInterval) - } - s.interval = d - - if len(s.Images) == 0 { - return fmt.Errorf("%s.images must not be empty", field) - } - for _, image := range s.Images { - if err := validateScheduledImage(image); err != nil { - return fmt.Errorf("invalid %s.images entry %q: %w", field, image, err) - } - } - for _, platform := range s.Platforms { - if err := validatePlatform(platform); err != nil { - return fmt.Errorf("invalid %s.platforms entry %q: %w", field, platform, err) - } - } - return nil -} - -func validateScheduledImage(image string) error { - p, err := purl.Parse(image) - if err != nil { - return err - } - if p.Type != "docker" && p.Type != "oci" { - return fmt.Errorf("only pkg:docker and pkg:oci images can be scheduled") - } - if p.Version == "" { - return fmt.Errorf("a tag is required") - } - repositoryURL := p.Qualifiers.Map()["repository_url"] - if p.Type == "oci" && repositoryURL == "" { - return fmt.Errorf("pkg:oci needs a repository_url qualifier") - } - // Only pkg:oci mirrors the tag qualifier next to the digest. - pinned := strings.HasPrefix(p.Version, "sha256:") - if pinned && (p.Type == "docker" || p.Qualifiers.Map()["tag"] == "") { - return fmt.Errorf("a digest never moves, so there is nothing to refresh; use a tag") - } - return nil -} - -// validatePlatform matches handler.ParsePlatform, which config cannot import. -func validatePlatform(value string) error { - os, rest, _ := strings.Cut(value, "/") - architecture, variant, _ := strings.Cut(rest, "/") - if os == "" || architecture == "" || strings.Contains(variant, "/") || strings.Contains(value, ",") { - return fmt.Errorf("want os/architecture[/variant]") - } - return nil -} diff --git a/internal/config/mirror_test.go b/internal/config/mirror_test.go deleted file mode 100644 index ae1220d..0000000 --- a/internal/config/mirror_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - "testing" - "time" -) - -func TestMirrorConfigValidate(t *testing.T) { - const digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000" - tests := []struct { - name string - schedule MirrorSchedule - wantErr bool - }{ - {name: "docker tags", schedule: MirrorSchedule{Images: []string{"pkg:docker/library/alpine@3.20", "pkg:docker/nginx@stable"}, Interval: "6h"}}, - {name: "oci with tag", schedule: MirrorSchedule{Images: []string{"pkg:oci/app@v1?repository_url=ghcr.io/acme/app"}, Interval: "1m"}}, - {name: "oci digest with tag", schedule: MirrorSchedule{Images: []string{"pkg:oci/app@" + digest + "?repository_url=ghcr.io/acme/app&tag=v1"}, Interval: "1h"}}, - {name: "platforms", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}, Interval: "1h", Platforms: []string{"linux/amd64", "linux/arm64/v8"}}}, - {name: "missing interval", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}}, wantErr: true}, - {name: "bad interval", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}, Interval: "often"}, wantErr: true}, - {name: "interval under a minute", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}, Interval: "30s"}, wantErr: true}, - {name: "no images", schedule: MirrorSchedule{Interval: "1h"}, wantErr: true}, - {name: "unparsable purl", schedule: MirrorSchedule{Images: []string{"alpine:3"}, Interval: "1h"}, wantErr: true}, - {name: "not an image", schedule: MirrorSchedule{Images: []string{"pkg:npm/lodash@4.17.21"}, Interval: "1h"}, wantErr: true}, - {name: "no tag", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine"}, Interval: "1h"}, wantErr: true}, - {name: "docker digest", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@" + digest}, Interval: "1h"}, wantErr: true}, - {name: "docker digest with tag qualifier", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@" + digest + "?tag=3"}, Interval: "1h"}, wantErr: true}, - {name: "oci digest only", schedule: MirrorSchedule{Images: []string{"pkg:oci/app@" + digest + "?repository_url=ghcr.io/acme/app"}, Interval: "1h"}, wantErr: true}, - {name: "oci without repository_url", schedule: MirrorSchedule{Images: []string{"pkg:oci/app@v1"}, Interval: "1h"}, wantErr: true}, - {name: "platform without architecture", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}, Interval: "1h", Platforms: []string{"linux"}}, wantErr: true}, - {name: "platform list in one entry", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}, Interval: "1h", Platforms: []string{"linux/amd64,linux/arm64"}}, wantErr: true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - m := MirrorConfig{Schedules: []MirrorSchedule{tt.schedule}} - err := m.Validate() - if (err != nil) != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestMirrorScheduleIntervalDuration(t *testing.T) { - m := MirrorConfig{Schedules: []MirrorSchedule{{Images: []string{"pkg:docker/alpine@3"}, Interval: "6h"}}} - if err := m.Validate(); err != nil { - t.Fatalf("Validate: %v", err) - } - if got := m.Schedules[0].IntervalDuration(); got != 6*time.Hour { - t.Errorf("IntervalDuration() = %v, want 6h", got) - } -} - -func TestValidateRejectsMirrorSchedulesWithoutCachedArtifacts(t *testing.T) { - cfg := Default() - cfg.Mirror.Schedules = []MirrorSchedule{{Images: []string{"pkg:docker/alpine@3"}, Interval: "1h"}} - cfg.Storage.CacheArtifacts = false - if err := cfg.Validate(); err == nil { - t.Error("Validate() accepted mirror.schedules with storage.cache_artifacts: false") - } -} - -func TestLoadYAMLMirrorSchedules(t *testing.T) { - path := filepath.Join(t.TempDir(), "config.yaml") - data := ` -mirror: - schedules: - - images: [pkg:docker/library/alpine@3.20] - interval: 6h - platforms: [linux/amd64] - - images: [pkg:docker/nginx@stable] - interval: 30s -` - if err := os.WriteFile(path, []byte(data), 0o600); err != nil { - t.Fatal(err) - } - cfg, err := Load(path) - if err != nil { - t.Fatalf("Load: %v", err) - } - if len(cfg.Mirror.Schedules) != 2 || cfg.Mirror.Schedules[0].Platforms[0] != "linux/amd64" { - t.Fatalf("schedules = %+v", cfg.Mirror.Schedules) - } - if err := cfg.Validate(); err == nil { - t.Error("Validate() accepted a 30s interval in the second schedule") - } -} diff --git a/internal/config/projects.go b/internal/config/projects.go deleted file mode 100644 index f416007..0000000 --- a/internal/config/projects.go +++ /dev/null @@ -1,53 +0,0 @@ -package config - -import ( - "fmt" - "time" -) - -// ProjectsConfig configures the project inventory: repositories upload an -// SBOM to /api/projects, and the proxy shows which of them use which package -// versions. -type ProjectsConfig struct { - // Enabled registers the /api/projects endpoints and the UI pages. Off by - // default. - Enabled bool `json:"enabled" yaml:"enabled"` - - // Token is the bearer token required to upload and delete projects. - // Required when Enabled. Supports ${VAR_NAME} expansion like AuthConfig - // fields. - Token string `json:"token" yaml:"token"` - - // MaxAge removes projects that have not uploaded an SBOM for this long - // (e.g., "90d"). Empty or "0" keeps them until they are deleted. - MaxAge string `json:"max_age" yaml:"max_age"` - - maxAge time.Duration -} - -// TokenExpanded returns Token with ${VAR_NAME} references expanded. -func (p *ProjectsConfig) TokenExpanded() string { - return expandEnv(p.Token) -} - -// MaxAgeDuration returns the parsed MaxAge, 0 to keep projects. Validate -// must run first. -func (p *ProjectsConfig) MaxAgeDuration() time.Duration { - return p.maxAge -} - -// Validate checks the project inventory settings. -func (p *ProjectsConfig) Validate() error { - p.maxAge = 0 - if p.MaxAge != "" { - d, err := parseRetentionDuration("projects.max_age", p.MaxAge) - if err != nil { - return err - } - p.maxAge = d - } - if p.Enabled && p.TokenExpanded() == "" { - return fmt.Errorf("projects.token is required when projects.enabled is true") - } - return nil -} diff --git a/internal/config/projects_test.go b/internal/config/projects_test.go deleted file mode 100644 index ce3c27c..0000000 --- a/internal/config/projects_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package config - -import ( - "testing" - "time" -) - -func TestProjectsConfigValidate(t *testing.T) { - t.Setenv("PROJECTS_TEST_TOKEN", "from-env") - - tests := []struct { - name string - projects ProjectsConfig - wantErr bool - maxAge time.Duration - }{ - {name: "disabled without token", projects: ProjectsConfig{}}, - {name: "enabled without token", projects: ProjectsConfig{Enabled: true}, wantErr: true}, - {name: "enabled with token", projects: ProjectsConfig{Enabled: true, Token: "s3cret"}}, - {name: "token from env reference", projects: ProjectsConfig{Enabled: true, Token: "${PROJECTS_TEST_TOKEN}"}}, - {name: "token env reference unset", projects: ProjectsConfig{Enabled: true, Token: "${PROJECTS_TEST_UNSET}"}, wantErr: true}, - {name: "max age in days", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "90d"}, maxAge: 90 * 24 * time.Hour}, - {name: "max age in hours", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "36h"}, maxAge: 36 * time.Hour}, - {name: "max age zero keeps projects", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "0"}}, - {name: "invalid max age", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "soon"}, wantErr: true}, - {name: "negative max age", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "-1d"}, wantErr: true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := tt.projects.Validate() - if (err != nil) != tt.wantErr { - t.Fatalf("Validate() error = %v, wantErr %v", err, tt.wantErr) - } - if got := tt.projects.MaxAgeDuration(); err == nil && got != tt.maxAge { - t.Errorf("MaxAgeDuration() = %v, want %v", got, tt.maxAge) - } - }) - } -} - -func TestLoadFromEnvProjects(t *testing.T) { - cfg := Default() - t.Setenv("PROXY_PROJECTS_ENABLED", "true") - t.Setenv("PROXY_PROJECTS_TOKEN", "env-token") - t.Setenv("PROXY_PROJECTS_MAX_AGE", "30d") - - cfg.LoadFromEnv() - - if !cfg.Projects.Enabled { - t.Error("Projects.Enabled not loaded from PROXY_PROJECTS_ENABLED") - } - if cfg.Projects.TokenExpanded() != "env-token" { - t.Errorf("Projects.Token = %q, want env-token", cfg.Projects.Token) - } - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate: %v", err) - } - if got := cfg.Projects.MaxAgeDuration(); got != 30*24*time.Hour { - t.Errorf("MaxAgeDuration() = %v, want 30 days", got) - } -} - -func TestValidateRejectsProjectsWithoutToken(t *testing.T) { - cfg := Default() - cfg.Projects = ProjectsConfig{Enabled: true} - if err := cfg.Validate(); err == nil { - t.Error("Validate() accepted projects.enabled without a token") - } -} diff --git a/internal/config/request_timeouts_test.go b/internal/config/request_timeouts_test.go deleted file mode 100644 index 01848c6..0000000 --- a/internal/config/request_timeouts_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -func TestRequestTimeouts(t *testing.T) { - for _, value := range []string{"", "0", "0s", "18m", "invalid", "-1s", "999999999999999999h"} { - t.Run(value, func(t *testing.T) { - cfg := Default() - cfg.WriteTimeout, cfg.ArtifactTimeout = value, value - valid := value == "" || value == "0" || value == "0s" || value == "18m" - want := 5 * time.Minute - if value == "0" || value == "0s" { - want = 0 - } - if value == "18m" { - want = 18 * time.Minute - } - if got := cfg.ParseWriteTimeout(); got != want { - t.Errorf("write timeout = %v, want %v", got, want) - } - if got := cfg.ParseArtifactTimeout(); got != want { - t.Errorf("artifact timeout = %v, want %v", got, want) - } - err := cfg.Validate() - if (err == nil) != valid { - t.Fatalf("Validate() = %v, valid = %v", err, valid) - } - if !valid { - if !strings.Contains(err.Error(), "write_timeout") { - t.Fatalf("error does not identify write_timeout: %v", err) - } - cfg.WriteTimeout = "" - if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "artifact_timeout") { - t.Fatalf("artifact validation = %v", err) - } - } - }) - } -} - -func TestLoadRequestTimeouts(t *testing.T) { - for _, format := range []struct{ extension, data string }{ - {"yaml", "write_timeout: 18m\nartifact_timeout: 10m\nhttp_timeout: 90s\n"}, - {"json", `{"write_timeout":"18m","artifact_timeout":"10m","http_timeout":"90s"}`}, - } { - t.Run(format.extension, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "config."+format.extension) - if err := os.WriteFile(path, []byte(format.data), 0600); err != nil { - t.Fatal(err) - } - cfg, err := Load(path) - if err != nil { - t.Fatal(err) - } - if cfg.ParseWriteTimeout() != 18*time.Minute || cfg.ParseArtifactTimeout() != 10*time.Minute || cfg.ParseHTTPTimeout() != 90*time.Second { - t.Fatalf("file timeouts = %v, %v, %v", cfg.WriteTimeout, cfg.ArtifactTimeout, cfg.HTTPTimeout) - } - t.Setenv("PROXY_WRITE_TIMEOUT", "20m") - t.Setenv("PROXY_ARTIFACT_TIMEOUT", "0") - t.Setenv("PROXY_HTTP_TIMEOUT", "15s") - cfg.LoadFromEnv() - if cfg.ParseWriteTimeout() != 20*time.Minute || cfg.ParseArtifactTimeout() != 0 || cfg.ParseHTTPTimeout() != 15*time.Second { - t.Fatalf("environment timeouts = %v, %v, %v", cfg.WriteTimeout, cfg.ArtifactTimeout, cfg.HTTPTimeout) - } - if err := cfg.Validate(); err != nil { - t.Fatal(err) - } - }) - } -} diff --git a/internal/config/retention.go b/internal/config/retention.go deleted file mode 100644 index 9493d28..0000000 --- a/internal/config/retention.go +++ /dev/null @@ -1,169 +0,0 @@ -package config - -import ( - "fmt" - "math" - "sort" - "strconv" - "strings" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/purl" -) - -// parseDayDuration parses a duration that may use a "d" suffix for days, as -// cooldown values do. Day counts that are not finite or exceed -// maxDurationDays are rejected, since converting them to a time.Duration -// would overflow. -func parseDayDuration(s string) (time.Duration, error) { - if num, ok := strings.CutSuffix(strings.TrimSpace(s), "d"); ok { - days, err := strconv.ParseFloat(num, 64) - if err == nil && (math.IsNaN(days) || math.IsInf(days, 0) || math.Abs(days) > maxDurationDays) { - return 0, fmt.Errorf("invalid duration %q: day count out of range", s) - } - } - return cooldown.ParseDuration(s) -} - -// parseRetentionDuration parses one retention value. field names the -// setting in error messages. -func parseRetentionDuration(field, value string) (time.Duration, error) { - d, err := parseDayDuration(value) - if err != nil { - return 0, fmt.Errorf("invalid %s: %w", field, err) - } - if d < 0 { - return 0, fmt.Errorf("invalid %s %q: must not be negative", field, value) - } - return d, nil -} - -// RetentionRules validates storage.retention against the ecosystems -// registered with reg and resolves it. An ecosystem or package that reg -// does not support is an error, so a rule is never silently ignored. -func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error) { - r := c.Storage.Retention - rules := retention.Rules{ - Ecosystems: map[string]time.Duration{}, - Packages: map[string]time.Duration{}, - Sources: map[string]map[string]time.Duration{}, - } - - var err error - if rules.Default, err = parseRetentionDuration("storage.retention.default", r.Default); err != nil { - return retention.Rules{}, err - } - - if r.SweepInterval != "" { - d, err := parseDayDuration(r.SweepInterval) - if err != nil { - return retention.Rules{}, fmt.Errorf("invalid storage.retention.sweep_interval: %w", err) - } - if d < minRetentionSweepInterval { - return retention.Rules{}, fmt.Errorf("invalid storage.retention.sweep_interval %q: must be at least %s", - r.SweepInterval, minRetentionSweepInterval) - } - } - - for _, key := range sortedKeys(r.Ecosystems) { - field := "storage.retention.ecosystems." + key - if !retention.IsKnown(key) { - return retention.Rules{}, fmt.Errorf("invalid %s: unknown ecosystem %q (valid: %s)", - field, key, strings.Join(retention.KnownKeys, ", ")) - } - if _, ok := reg.Lookup(key); !ok { - return retention.Rules{}, fmt.Errorf("invalid %s: retention is not supported for %q in this version", field, key) - } - d, err := parseRetentionDuration(field, r.Ecosystems[key]) - if err != nil { - return retention.Rules{}, err - } - rules.Ecosystems[key] = d - } - - for _, key := range sortedKeys(r.Packages) { - field := fmt.Sprintf("storage.retention.packages[%q]", key) - canonical, sourceOf, err := retentionPackageKey(reg, key) - if err != nil { - return retention.Rules{}, fmt.Errorf("invalid %s: %w", field, err) - } - d, err := parseRetentionDuration(field, r.Packages[key]) - if err != nil { - return retention.Rules{}, err - } - if sourceOf != "" { - if rules.Sources[sourceOf] == nil { - rules.Sources[sourceOf] = map[string]time.Duration{} - } - rules.Sources[sourceOf][key] = d - continue - } - if prev, ok := rules.Packages[canonical]; ok && prev != d { - return retention.Rules{}, fmt.Errorf("invalid %s: conflicts with another key for package %s", field, canonical) - } - rules.Packages[canonical] = d - } - - return rules, nil -} - -// retentionPackageKey returns the stored package PURL a configured package -// key stands for, or, for a key that has to be resolved through the upstream -// first, the ecosystem key it belongs to. -func retentionPackageKey(reg *retention.Registry, key string) (canonical, sourceOf string, _ error) { - // A julia PURL needs a uuid qualifier to parse at all, and the julia - // handler stores none, so no key could ever match one of its packages. - if strings.HasPrefix(strings.ToLower(key), "pkg:julia/") { - return "", "", fmt.Errorf("package rules are not supported for julia") - } - p, err := purl.Parse(key) - if err != nil { - return "", "", fmt.Errorf("not a valid package PURL: %w", err) - } - if p.Version != "" { - return "", "", fmt.Errorf("package PURL must not carry a version") - } - // A rule names a whole package; qualifiers and subpaths would be - // dropped on the way to the stored PURL, so refuse them rather than - // let "?type=pom" quietly cover every file of the package. - if len(p.Qualifiers) > 0 || p.Subpath != "" { - return "", "", fmt.Errorf("package PURL must not carry qualifiers or a subpath") - } - if canonical, ok := reg.Canonical(p); ok { - return canonical, "", nil - } - if ecosystem, ok := reg.Source(p); ok { - return "", ecosystem, nil - } - ecosystem, registered := reg.KeyForDBEcosystem(purl.PURLTypeToEcosystem(p.Type)) - if !registered { - return "", "", fmt.Errorf("retention is not supported for %s packages in this version", p.Type) - } - if spec, _ := reg.Lookup(ecosystem); spec.CanonicalPackage == nil { - return "", "", fmt.Errorf("package rules are not supported for %s; use storage.retention.ecosystems.%s", ecosystem, ecosystem) - } - return "", "", fmt.Errorf("does not match how %s packages are stored; see the package rule forms in docs/configuration.md", ecosystem) -} - -func sortedKeys(m map[string]string) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - -// ParseRetentionSweepInterval returns how often the retention sweep runs. -func (c *Config) ParseRetentionSweepInterval() time.Duration { - if c.Storage.Retention.SweepInterval == "" { - return defaultRetentionSweepInterval - } - d, err := parseDayDuration(c.Storage.Retention.SweepInterval) - if err != nil || d < minRetentionSweepInterval { - return defaultRetentionSweepInterval - } - return d -} diff --git a/internal/config/retention_test.go b/internal/config/retention_test.go deleted file mode 100644 index eb614e8..0000000 --- a/internal/config/retention_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package config - -import ( - "strings" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/retention" -) - -const retentionDay = 24 * time.Hour - -func retentionTestRegistry() *retention.Registry { - reg := retention.NewRegistry() - reg.Register(retention.Spec{Key: "generic", CanonicalPackage: retention.DefaultCanonical("generic")}) - reg.Register(retention.Spec{Key: "npm", CanonicalPackage: retention.DefaultCanonical("npm")}) - reg.Register(retention.Spec{Key: "julia"}) - reg.Register(retention.Spec{Key: "helm"}) - return reg -} - -func TestRetentionRulesUnsetIsDisabled(t *testing.T) { - rules, err := Default().RetentionRules(retentionTestRegistry()) - if err != nil { - t.Fatalf("RetentionRules() error = %v", err) - } - if got := rules.MinPositive(); got != 0 { - t.Errorf("MinPositive() = %v, want 0 for an unset retention block", got) - } -} - -func TestRetentionRulesResolves(t *testing.T) { - cfg := Default() - cfg.Storage.Retention = RetentionConfig{ - Default: "30d", - Ecosystems: map[string]string{"npm": "14d"}, - Packages: map[string]string{ - "pkg:npm/lodash": "0", - "pkg:npm/@babel/core": "90d", - "pkg:generic/jqlang%2Fjq": "7d", - }, - SweepInterval: "5m", - } - - rules, err := cfg.RetentionRules(retentionTestRegistry()) - if err != nil { - t.Fatalf("RetentionRules() error = %v", err) - } - if rules.Default != 30*retentionDay { - t.Errorf("Default = %v, want 30d", rules.Default) - } - if got := rules.For("npm", "pkg:npm/left-pad"); got != 14*retentionDay { - t.Errorf("npm rule = %v, want 14d", got) - } - if got, ok := rules.Packages["pkg:npm/lodash"]; !ok || got != 0 { - t.Errorf("lodash rule = %v, %v, want 0, true", got, ok) - } - if got := rules.For("npm", "pkg:npm/%40babel/core"); got != 90*retentionDay { - t.Errorf("babel rule = %v, want 90d under its canonical key", got) - } - if got := rules.For("generic", "pkg:generic/jqlang%2Fjq"); got != 7*retentionDay { - t.Errorf("generic package rule = %v, want 7d", got) - } - if got := rules.For("generic", "pkg:generic/other%2Frepo"); got != 0 { - t.Errorf("unconfigured generic package rule = %v, want 0 despite the global default", got) - } - if got := cfg.ParseRetentionSweepInterval(); got != 5*time.Minute { - t.Errorf("ParseRetentionSweepInterval() = %v, want 5m", got) - } -} - -func TestRetentionRulesErrors(t *testing.T) { - tests := []struct { - name string - modify func(*RetentionConfig) - wantErr string - }{ - {"bad default", func(r *RetentionConfig) { r.Default = "7x" }, "storage.retention.default"}, - {"negative default", func(r *RetentionConfig) { r.Default = "-1d" }, "must not be negative"}, - {"NaN days", func(r *RetentionConfig) { r.Default = "NaNd" }, "out of range"}, - {"infinite days", func(r *RetentionConfig) { r.Default = "Infd" }, "out of range"}, - {"overflowing days", func(r *RetentionConfig) { r.Default = "1e9d" }, "out of range"}, - {"zero sweep interval", func(r *RetentionConfig) { r.SweepInterval = "0s" }, "must be at least 1m0s"}, - {"sweep interval below a minute", func(r *RetentionConfig) { r.SweepInterval = "30s" }, "must be at least 1m0s"}, - {"bad sweep interval", func(r *RetentionConfig) { r.SweepInterval = "often" }, "sweep_interval"}, - {"unknown ecosystem", func(r *RetentionConfig) { r.Ecosystems = map[string]string{"nmp": "1d"} }, "unknown ecosystem"}, - {"ecosystem not registered", func(r *RetentionConfig) { r.Ecosystems = map[string]string{"oci": "14d"} }, - `storage.retention.ecosystems.oci: retention is not supported for "oci"`}, - {"bad ecosystem duration", func(r *RetentionConfig) { r.Ecosystems = map[string]string{"npm": "soon"} }, "storage.retention.ecosystems.npm"}, - {"package of unregistered ecosystem", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:cargo/serde": "1d"} }, - "not supported for cargo packages"}, - {"julia package", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:julia/Example": "1d"} }, - "not supported for julia"}, - {"package of ecosystem without package rules", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:helm/nginx": "1d"} }, - "package rules are not supported for helm; use storage.retention.ecosystems.helm"}, - {"invalid package PURL", func(r *RetentionConfig) { r.Packages = map[string]string{"lodash": "1d"} }, "not a valid package PURL"}, - {"npm package without its scope marker", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:npm/babel/core": "1d"} }, - "does not match how npm packages are stored"}, - {"npm package with an encoded scope separator", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:npm/%40babel%2Fcore": "1d"} }, - "does not match how npm packages are stored"}, - {"versioned package PURL", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:npm/lodash@4.17.21": "1d"} }, - "must not carry a version"}, - {"conflicting package keys", func(r *RetentionConfig) { - r.Packages = map[string]string{"pkg:npm/%40babel/core": "1d", "pkg:npm/@babel/core": "2d"} - }, "conflicts with another key"}, - {"package PURL with qualifiers", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:npm/lodash?type=tgz": "1d"} }, - "must not carry qualifiers or a subpath"}, - {"package PURL with subpath", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:npm/lodash#lib": "1d"} }, - "must not carry qualifiers or a subpath"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := Default() - tt.modify(&cfg.Storage.Retention) - _, err := cfg.RetentionRules(retentionTestRegistry()) - if err == nil { - t.Fatalf("RetentionRules() succeeded, want error containing %q", tt.wantErr) - } - if !strings.Contains(err.Error(), tt.wantErr) { - t.Errorf("RetentionRules() error = %q, want it to contain %q", err, tt.wantErr) - } - }) - } -} - -// Validate checks retention against the handlers' registry, which this -// package's tests leave empty, so any ecosystem rule fails while a bare -// default passes. -func TestValidateRetention(t *testing.T) { - cfg := Default() - cfg.Storage.Retention.Default = "30d" - if err := cfg.Validate(); err != nil { - t.Errorf("Validate() with only a default = %v, want nil", err) - } - - cfg.Storage.Retention.Ecosystems = map[string]string{"npm": "14d"} - err := cfg.Validate() - if err == nil || !strings.Contains(err.Error(), "not supported") { - t.Errorf("Validate() with an unsupported ecosystem = %v, want a not supported error", err) - } -} - -func TestRetentionEnvOverrides(t *testing.T) { - t.Setenv("PROXY_STORAGE_RETENTION_DEFAULT", "21d") - t.Setenv("PROXY_STORAGE_RETENTION_SWEEP_INTERVAL", "1h") - - cfg := Default() - cfg.LoadFromEnv() - - if cfg.Storage.Retention.Default != "21d" { - t.Errorf("Retention.Default = %q, want 21d", cfg.Storage.Retention.Default) - } - if got := cfg.ParseRetentionSweepInterval(); got != time.Hour { - t.Errorf("ParseRetentionSweepInterval() = %v, want 1h", got) - } -} - -func TestRetentionSweepIntervalDefault(t *testing.T) { - cfg := Default() - if got := cfg.ParseRetentionSweepInterval(); got != 10*time.Minute { - t.Errorf("ParseRetentionSweepInterval() = %v, want 10m", got) - } -} - -func TestGradleBuildCacheMaxAgeAcceptsDays(t *testing.T) { - cfg := Default() - cfg.Gradle.BuildCache.MaxAge = "7d" - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate() with max_age 7d = %v, want nil", err) - } - if got := cfg.ParseGradleBuildCacheMaxAge(); got != 7*retentionDay { - t.Errorf("ParseGradleBuildCacheMaxAge() = %v, want 168h", got) - } -} diff --git a/internal/database/analytics_test.go b/internal/database/analytics_test.go deleted file mode 100644 index 5084acd..0000000 --- a/internal/database/analytics_test.go +++ /dev/null @@ -1,464 +0,0 @@ -package database - -import ( - "database/sql" - "path/filepath" - "testing" - "time" - - "github.com/git-pkgs/purl" -) - -// seedArtifact inserts a package, a version and a cached artifact so that the -// ecosystem aggregation has a complete join path to walk. -func seedArtifact(t *testing.T, db *DB, ecosystem, name, version string, size, hits int64) { - t.Helper() - - pkgPURL := "pkg:" + ecosystem + "/" + name - versionPURL := pkgPURL + "@" + version - - if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: name}); err != nil { - t.Fatalf("UpsertPackage(%s): %v", pkgPURL, err) - } - if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { - t.Fatalf("UpsertVersion(%s): %v", versionPURL, err) - } - if err := db.UpsertArtifact(&Artifact{ - VersionPURL: versionPURL, - Filename: name + "-" + version + ".tgz", - UpstreamURL: "https://example.test/" + name, - StoragePath: sql.NullString{String: "objects/" + name, Valid: true}, - Size: sql.NullInt64{Int64: size, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - HitCount: hits, - }); err != nil { - t.Fatalf("UpsertArtifact(%s): %v", versionPURL, err) - } -} - -func newTestDB(t *testing.T) *DB { - t.Helper() - db, err := Create(filepath.Join(t.TempDir(), "analytics.db")) - if err != nil { - t.Fatalf("Create: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - return db -} - -func TestGetEcosystemStats(t *testing.T) { - db := newTestDB(t) - - // npm: 2 artifacts across 2 packages, 1000*3 + 500*1 = 3500 bytes served. - seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 3) - seedArtifact(t, db, "npm", "express", "4.18.2", 500, 1) - // cargo: a single heavily-hit artifact, 200*10 = 2000 bytes served. - seedArtifact(t, db, "cargo", "serde", "1.0.0", 200, 10) - - stats, err := db.GetEcosystemStats() - if err != nil { - t.Fatalf("GetEcosystemStats: %v", err) - } - if len(stats) != 2 { - t.Fatalf("expected 2 ecosystems, got %d: %+v", len(stats), stats) - } - - // Ordered by accumulated download volume, so npm (3500) precedes cargo (2000). - npm, cargo := stats[0], stats[1] - if npm.Ecosystem != "npm" || cargo.Ecosystem != "cargo" { - t.Fatalf("expected npm then cargo, got %q then %q", npm.Ecosystem, cargo.Ecosystem) - } - - if npm.DownloadedBytes != 3500 { - t.Errorf("npm DownloadedBytes = %d, want 3500", npm.DownloadedBytes) - } - if npm.Downloads != 4 { - t.Errorf("npm Downloads = %d, want 4", npm.Downloads) - } - if npm.CacheSize != 1500 { - t.Errorf("npm CacheSize = %d, want 1500", npm.CacheSize) - } - if npm.Artifacts != 2 { - t.Errorf("npm Artifacts = %d, want 2", npm.Artifacts) - } - if npm.Packages != 2 { - t.Errorf("npm Packages = %d, want 2", npm.Packages) - } - if npm.Versions != 2 { - t.Errorf("npm Versions = %d, want 2", npm.Versions) - } - - if cargo.DownloadedBytes != 2000 { - t.Errorf("cargo DownloadedBytes = %d, want 2000", cargo.DownloadedBytes) - } - if cargo.Downloads != 10 { - t.Errorf("cargo Downloads = %d, want 10", cargo.Downloads) - } -} - -func TestGetEcosystemStatsEmptyDatabase(t *testing.T) { - stats, err := newTestDB(t).GetEcosystemStats() - if err != nil { - t.Fatalf("GetEcosystemStats: %v", err) - } - if len(stats) != 0 { - t.Errorf("expected no ecosystems, got %+v", stats) - } -} - -// An ecosystem known from metadata but with nothing cached should still appear, -// so the analytics table can show it as idle rather than omitting it. -func TestGetEcosystemStatsIncludesEcosystemsWithoutArtifacts(t *testing.T) { - db := newTestDB(t) - seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) - - if err := db.UpsertPackage(&Package{PURL: "pkg:gem/rails", Ecosystem: "gem", Name: "rails"}); err != nil { - t.Fatalf("UpsertPackage: %v", err) - } - - stats, err := db.GetEcosystemStats() - if err != nil { - t.Fatalf("GetEcosystemStats: %v", err) - } - if len(stats) != 2 { - t.Fatalf("expected 2 ecosystems, got %d: %+v", len(stats), stats) - } - - // gem has no download volume, so it sorts last. - gem := stats[1] - if gem.Ecosystem != "gem" { - t.Fatalf("expected gem last, got %q", gem.Ecosystem) - } - if gem.Packages != 1 { - t.Errorf("gem Packages = %d, want 1", gem.Packages) - } - if gem.Artifacts != 0 || gem.CacheSize != 0 || gem.DownloadedBytes != 0 { - t.Errorf("expected gem to report no cached artifacts, got %+v", gem) - } -} - -// Eviction clears storage_path and size, which drops the artifact's historical -// hits out of the accumulated total. Pinning that here so the documented -// behaviour of GetEcosystemStats does not drift. -func TestGetEcosystemStatsExcludesEvictedArtifacts(t *testing.T) { - db := newTestDB(t) - seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 3) - seedArtifact(t, db, "npm", "express", "4.18.2", 500, 2) - - cleared, err := db.ClearArtifactCache("pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz", "objects/lodash") - if err != nil { - t.Fatalf("ClearArtifactCache: %v", err) - } - if !cleared { - t.Fatal("ClearArtifactCache cleared nothing") - } - - stats, err := db.GetEcosystemStats() - if err != nil { - t.Fatalf("GetEcosystemStats: %v", err) - } - if len(stats) != 1 { - t.Fatalf("expected 1 ecosystem, got %d: %+v", len(stats), stats) - } - - // Only express remains cached: 500 * 2 = 1000. - if got := stats[0].DownloadedBytes; got != 1000 { - t.Errorf("DownloadedBytes = %d, want 1000 (evicted artifact must not count)", got) - } - if got := stats[0].Artifacts; got != 1 { - t.Errorf("Artifacts = %d, want 1", got) - } - // The package row survives eviction, so both packages still count. - if got := stats[0].Packages; got != 2 { - t.Errorf("Packages = %d, want 2", got) - } -} - -func TestSortEcosystemStatsTieBreaks(t *testing.T) { - stats := []EcosystemStats{ - {Ecosystem: "zzz"}, - {Ecosystem: "aaa"}, - {Ecosystem: "mid", CacheSize: 10}, - {Ecosystem: "top", DownloadedBytes: 5}, - } - sortEcosystemStats(stats) - - want := []string{"top", "mid", "aaa", "zzz"} - for i, name := range want { - if stats[i].Ecosystem != name { - t.Errorf("position %d = %q, want %q (full order: %+v)", i, stats[i].Ecosystem, name, stats) - } - } -} - -// A cached artifact whose version row is missing must still be counted, so the -// per-ecosystem totals reconcile with GetTotalCacheSize rather than quietly -// coming up short. Nothing at the schema level enforces the link. -func TestGetEcosystemStatsCountsUnattributedArtifacts(t *testing.T) { - db := newTestDB(t) - seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) - - // An artifact pointing at a version that was never recorded. - if err := db.UpsertArtifact(&Artifact{ - VersionPURL: "pkg:npm/orphan@9.9.9", - Filename: "orphan-9.9.9.tgz", - UpstreamURL: "https://example.test/orphan", - StoragePath: sql.NullString{String: "objects/orphan", Valid: true}, - Size: sql.NullInt64{Int64: 500, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - HitCount: 4, - }); err != nil { - t.Fatalf("UpsertArtifact: %v", err) - } - - stats, err := db.GetEcosystemStats() - if err != nil { - t.Fatalf("GetEcosystemStats: %v", err) - } - - var cacheSize, artifacts, downloaded int64 - var sawUnattributed bool - for _, e := range stats { - cacheSize += e.CacheSize - artifacts += e.Artifacts - downloaded += e.DownloadedBytes - if e.Ecosystem == unattributedEcosystem { - sawUnattributed = true - } - } - - if !sawUnattributed { - t.Errorf("the orphaned artifact was dropped; got %+v", stats) - } - - // These must match what the unjoined queries report, or the UI shows two - // totals that do not add up. - wantSize, err := db.GetTotalCacheSize() - if err != nil { - t.Fatalf("GetTotalCacheSize: %v", err) - } - if cacheSize != wantSize { - t.Errorf("per-ecosystem cache size sums to %d, but GetTotalCacheSize reports %d", cacheSize, wantSize) - } - - wantCount, err := db.GetCachedArtifactCount() - if err != nil { - t.Fatalf("GetCachedArtifactCount: %v", err) - } - if artifacts != wantCount { - t.Errorf("per-ecosystem artifacts sum to %d, but GetCachedArtifactCount reports %d", artifacts, wantCount) - } - - // 1000*2 from lodash plus 500*4 from the orphan. - if downloaded != 4000 { - t.Errorf("downloaded bytes = %d, want 4000", downloaded) - } -} - -// The proxy writes "gem" and git-pkgs writes "rubygems". A database that has -// seen both must report one ecosystem, not two rows splitting its share. -func TestGetEcosystemStatsMergesAliasedEcosystems(t *testing.T) { - db := newTestDB(t) - seedArtifact(t, db, "gem", "colorize", "1.1.0", 1000, 3) - seedArtifact(t, db, "rubygems", "rails", "7.1.0", 4000, 2) - seedArtifact(t, db, "npm", "lodash", "4.17.21", 500, 1) - - stats, err := db.GetEcosystemStats() - if err != nil { - t.Fatalf("GetEcosystemStats: %v", err) - } - if len(stats) != 2 { - t.Fatalf("expected gem and rubygems merged into one row alongside npm, got %+v", stats) - } - - var ruby *EcosystemStats - for i := range stats { - if purl.NormalizeEcosystem(stats[i].Ecosystem) == "rubygems" { - ruby = &stats[i] - } - } - if ruby == nil { - t.Fatal("no row for the rubygems ecosystem") - } - - // 1000*3 + 4000*2 = 11000 - if ruby.DownloadedBytes != 11000 { - t.Errorf("DownloadedBytes = %d, want 11000", ruby.DownloadedBytes) - } - if ruby.CacheSize != 5000 { - t.Errorf("CacheSize = %d, want 5000", ruby.CacheSize) - } - if ruby.Artifacts != 2 || ruby.Packages != 2 { - t.Errorf("Artifacts=%d Packages=%d, want 2 and 2", ruby.Artifacts, ruby.Packages) - } - - // The surviving row keeps the spelling with the most cached bytes, because - // the UI filters and links by it and the packages table stores it raw. - if ruby.Ecosystem != "rubygems" { - t.Errorf("Ecosystem = %q, want the dominant raw spelling %q", ruby.Ecosystem, "rubygems") - } -} - -// A version whose package row is missing must be bucketed like an orphaned -// artifact, so the per-ecosystem figures reconcile with GetCacheStats. -func TestGetEcosystemStatsCountsUnattributedVersions(t *testing.T) { - db := newTestDB(t) - seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 1) - if err := db.UpsertVersion(&Version{PURL: "pkg:npm/ghost@1.0.0", PackagePURL: "pkg:npm/ghost"}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - - stats, err := db.GetEcosystemStats() - if err != nil { - t.Fatalf("GetEcosystemStats: %v", err) - } - - var versions int64 - for _, e := range stats { - versions += e.Versions - } - - cacheStats, err := db.GetCacheStats() - if err != nil { - t.Fatalf("GetCacheStats: %v", err) - } - if versions != cacheStats.TotalVersions { - t.Errorf("per-ecosystem versions sum to %d, but GetCacheStats reports %d", - versions, cacheStats.TotalVersions) - } -} - -// The surviving spelling must be the one with the most cached bytes whatever -// order the rows arrive in. GetEcosystemStats builds its input by ranging a -// map, so an implementation that compared against the running total instead of -// each row's own size would pick a different name between refreshes — flipping -// the analytics row's badge and its /ui/packages filter link. -func TestMergeAliasedEcosystemsPicksLargestWhateverTheOrder(t *testing.T) { - rows := []EcosystemStats{ - {Ecosystem: "gem", CacheSize: 5, Artifacts: 1}, - {Ecosystem: "rubygems", CacheSize: 4, Artifacts: 1}, - {Ecosystem: "RubyGems", CacheSize: 6, Artifacts: 1}, - } - - for _, order := range [][]int{ - {0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}, - } { - in := make([]EcosystemStats, 0, len(order)) - for _, i := range order { - in = append(in, rows[i]) - } - - got := mergeAliasedEcosystems(in) - if len(got) != 1 { - t.Fatalf("order %v: got %d rows, want 1", order, len(got)) - } - if got[0].Ecosystem != "RubyGems" { - t.Errorf("order %v: Ecosystem = %q, want the largest spelling %q", - order, got[0].Ecosystem, "RubyGems") - } - if got[0].CacheSize != 15 { - t.Errorf("order %v: CacheSize = %d, want 15", order, got[0].CacheSize) - } - } -} - -// Equal sizes still have to resolve to one answer, for the same reason. -func TestMergeAliasedEcosystemsBreaksSizeTiesByName(t *testing.T) { - a := []EcosystemStats{{Ecosystem: "gem", CacheSize: 5}, {Ecosystem: "rubygems", CacheSize: 5}} - b := []EcosystemStats{{Ecosystem: "rubygems", CacheSize: 5}, {Ecosystem: "gem", CacheSize: 5}} - - first := mergeAliasedEcosystems(a)[0].Ecosystem - second := mergeAliasedEcosystems(b)[0].Ecosystem - if first != second { - t.Errorf("input order changed the surviving spelling: %q vs %q", first, second) - } -} - -// Cooldown stores publish times of versions nobody downloaded, without a -// package row. They count under the ecosystem of their package PURL, not as -// unattributed. -func TestGetEcosystemStatsAttributesPublishTimesByPURL(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) - published := time.Now().Add(-time.Hour) - for _, v := range []struct{ version, pkg string }{ - {"pkg:npm/react@19.0.0", "pkg:npm/react"}, - {"pkg:golang/github.com/swaggo/swag@v1.16.3", "pkg:golang/github.com/swaggo/swag"}, - {"pkg:golang/github.com/swaggo/swag@v1.16.4", "pkg:golang/github.com/swaggo/swag"}, - {"pkg:composer/vendor/pkg@1.0.0", "pkg:composer/vendor/pkg"}, - {"not-a-purl@1", "not-a-purl"}, - } { - if err := db.SetVersionPublishedAt(v.version, v.pkg, published); err != nil { - t.Fatal(err) - } - } - - stats, err := db.GetEcosystemStats() - if err != nil { - t.Fatalf("GetEcosystemStats: %v", err) - } - versions := map[string]int64{} - for _, e := range stats { - versions[e.Ecosystem] = e.Versions - } - want := map[string]int64{"npm": 2, "golang": 2, "composer": 1, unattributedEcosystem: 1} - for ecosystem, n := range want { - if versions[ecosystem] != n { - t.Errorf("versions of %s = %d, want %d (all: %v)", ecosystem, versions[ecosystem], n, versions) - } - } - if len(versions) != len(want) { - t.Errorf("ecosystems = %v, want %v", versions, want) - } - }) -} - -// TestGetEcosystemStatsVersionsReconcile checks that the versions counted -// per ecosystem add up to the cache stats' total, with publish times stored -// under namespaced PURLs, an alias of a package row's ecosystem, and a PURL -// that can't be parsed. -func TestGetEcosystemStatsVersionsReconcile(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) - seedArtifact(t, db, "go", "github.com/pkg/errors", "v0.9.1", 500, 1) - published := time.Now().Add(-time.Hour) - for _, v := range []struct{ version, pkg string }{ - {"pkg:npm/%40babel/core@7.25.2", "pkg:npm/%40babel/core"}, - {"pkg:maven/org.apache.commons/commons-lang3@3.14.0", "pkg:maven/org.apache.commons/commons-lang3"}, - {"pkg:pypi/requests@2.32.3", "pkg:pypi/requests"}, - {"pkg:golang/github.com/swaggo/swag@v1.16.4", "pkg:golang/github.com/swaggo/swag"}, - {"~odd@1", "~odd"}, - } { - if err := db.SetVersionPublishedAt(v.version, v.pkg, published); err != nil { - t.Fatal(err) - } - } - - stats, err := db.GetEcosystemStats() - if err != nil { - t.Fatalf("GetEcosystemStats: %v", err) - } - versions := map[string]int64{} - var sum int64 - for _, e := range stats { - versions[e.Ecosystem] += e.Versions - sum += e.Versions - } - want := map[string]int64{"npm": 2, "go": 2, "maven": 1, "pypi": 1, unattributedEcosystem: 1} - if len(versions) != len(want) { - t.Errorf("ecosystems = %v, want %v", versions, want) - } - for ecosystem, n := range want { - if versions[ecosystem] != n { - t.Errorf("versions of %s = %d, want %d (all: %v)", ecosystem, versions[ecosystem], n, versions) - } - } - cache, err := db.GetCacheStats() - if err != nil { - t.Fatal(err) - } - if sum != cache.TotalVersions { - t.Errorf("versions per ecosystem add up to %d, cache stats count %d", sum, cache.TotalVersions) - } - }) -} diff --git a/internal/database/cache_eviction.go b/internal/database/cache_eviction.go deleted file mode 100644 index 33bcdae..0000000 --- a/internal/database/cache_eviction.go +++ /dev/null @@ -1,247 +0,0 @@ -package database - -import ( - "database/sql" - "slices" - "time" -) - -// Kinds of CacheEntry. -const ( - CacheEntryArtifact = "artifact" - CacheEntryMetadata = "metadata" -) - -// CacheEntry is a cached object size eviction can remove: an artifact fetched -// from upstream or a metadata document. For an artifact Key is its version -// PURL and Name its filename; for metadata Key is the ecosystem and Name the -// cache key. FetchedAt is only read for artifacts, UpdatedAt for metadata -// not backfilled yet. -type CacheEntry struct { - Kind string `db:"kind"` - Key string `db:"cache_key"` - Name string `db:"name"` - StoragePath sql.NullString `db:"storage_path"` - Size sql.NullInt64 `db:"size"` - LastAccessedAt sql.NullTime `db:"last_accessed_at"` - FetchedAt sql.NullTime `db:"fetched_at"` - UpdatedAt sql.NullTime `db:"updated_at"` -} - -// GetEvictableCacheSize returns the bytes max_size limits: cached upstream -// artifacts and cached metadata documents. -func (db *DB) GetEvictableCacheSize() (int64, error) { - var total int64 - err := db.Get(&total, ` - SELECT (SELECT COALESCE(SUM(size), 0) FROM artifacts WHERE storage_path IS NOT NULL AND source <> 'local') - + (SELECT COALESCE(SUM(size), 0) FROM metadata_cache)`) - return total, err -} - -// GetLeastRecentlyUsedCacheEntries returns up to limit artifacts and metadata -// documents, least recently used first. An artifact counts as used when it -// was fetched until it is first served, so a fresh download does not go -// before every metadata document. Each kind is read in order on its own and -// the two are merged here, which keeps the queries simple for both dialects. -func (db *DB) GetLeastRecentlyUsedCacheEntries(limit int) ([]CacheEntry, error) { - var artifacts, metadata []CacheEntry - if err := db.Select(&artifacts, db.Rebind(` - SELECT 'artifact' AS kind, version_purl AS cache_key, filename AS name, - storage_path, size, last_accessed_at, fetched_at - FROM artifacts - WHERE storage_path IS NOT NULL AND source <> 'local' - ORDER BY COALESCE(last_accessed_at, fetched_at) ASC NULLS FIRST - LIMIT ? - `), limit); err != nil { - return nil, err - } - if err := db.Select(&metadata, db.Rebind(` - SELECT 'metadata' AS kind, ecosystem AS cache_key, name, - storage_path, size, last_accessed_at, updated_at - FROM metadata_cache - ORDER BY COALESCE(last_accessed_at, updated_at) ASC NULLS FIRST - LIMIT ? - `), limit); err != nil { - return nil, err - } - // Rows not backfilled yet fall back to updated_at, which Postgres keeps - // in local time, so the order is fixed up once it is read as UTC. - for i := range metadata { - metadata[i].UpdatedAt = db.writtenAtUTC(metadata[i].UpdatedAt) - } - slices.SortStableFunc(metadata, func(a, b CacheEntry) int { - switch { - case a.usedAfter(b): - return 1 - case b.usedAfter(a): - return -1 - } - return 0 - }) - - entries := make([]CacheEntry, 0, min(limit, len(artifacts)+len(metadata))) - for len(entries) < limit && (len(artifacts) > 0 || len(metadata) > 0) { - if len(metadata) == 0 || (len(artifacts) > 0 && !artifacts[0].usedAfter(metadata[0])) { - entries, artifacts = append(entries, artifacts[0]), artifacts[1:] - } else { - entries, metadata = append(entries, metadata[0]), metadata[1:] - } - } - return entries, nil -} - -// lastUsed is when e was last served, or for an artifact not served since, -// when it was fetched. It is invalid when neither is known. -func (e CacheEntry) lastUsed() sql.NullTime { - if e.LastAccessedAt.Valid { - return e.LastAccessedAt - } - if e.FetchedAt.Valid { - return e.FetchedAt - } - return e.UpdatedAt -} - -// usedAfter reports whether e was used later than other. Entries with no -// time known count as the oldest, as NULLS FIRST orders them. -func (e CacheEntry) usedAfter(other CacheEntry) bool { - a, b := e.lastUsed(), other.lastUsed() - if !a.Valid { - return false - } - return !b.Valid || a.Time.After(b.Time) -} - -// EvictMetadataCache deletes a metadata cache row if it still points at -// storagePath and has not been used since lastAccessed, queueing the path in -// the same transaction so a crash before the blob is deleted cannot orphan it. -func (db *DB) EvictMetadataCache(ecosystem, name, storagePath string, lastAccessed sql.NullTime) (bool, error) { - query := ` - DELETE FROM metadata_cache - WHERE ecosystem = ? AND name = ? AND storage_path = ? AND last_accessed_at IS NULL` - args := []any{ecosystem, name, storagePath} - if lastAccessed.Valid { - query = ` - DELETE FROM metadata_cache - WHERE ecosystem = ? AND name = ? AND storage_path = ? - AND (last_accessed_at IS NULL OR last_accessed_at <= ?)` - args = append(args, lastAccessed.Time) - } - tx, err := db.Beginx() - if err != nil { - return false, err - } - defer func() { _ = tx.Rollback() }() - res, err := tx.Exec(db.Rebind(query), args...) - if err != nil { - return false, err - } - if n, err := res.RowsAffected(); err != nil || n == 0 { - return false, err - } - if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), storagePath, time.Now().UTC()); err != nil { - return false, err - } - return true, tx.Commit() -} - -// DiscardMetadataCache deletes a metadata cache row whose stored document -// failed its integrity check, if it still points at storagePath with digest, -// and queues the path for deletion in the same transaction. A row a newer -// fetch has rewritten is left alone. -func (db *DB) DiscardMetadataCache(ecosystem, name, storagePath, digest string) error { - tx, err := db.Beginx() - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - res, err := tx.Exec(db.Rebind(` - DELETE FROM metadata_cache - WHERE ecosystem = ? AND name = ? AND storage_path = ? AND content_digest = ?`), - ecosystem, name, storagePath, digest) - if err != nil { - return err - } - if n, err := res.RowsAffected(); err != nil || n == 0 { - return err - } - if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), storagePath, time.Now().UTC()); err != nil { - return err - } - return tx.Commit() -} - -// RecordMetadataHit marks a cached metadata document as served now. Like -// RecordArtifactHit it is buffered when BatchHits is on. -func (db *DB) RecordMetadataHit(ecosystem, name string) error { - now := time.Now() - if db.hits != nil { - db.hits.addMetadata(metadataHitKey{ecosystem, name}, now) - return nil - } - _, err := db.Exec(db.Rebind(recordMetadataHitQuery), now.UTC(), now.UTC(), ecosystem, name) - return err -} - -// recordMetadataHitQuery takes the access time twice, then the ecosystem and -// key. The time only moves forward, as for artifacts. -const recordMetadataHitQuery = ` - UPDATE metadata_cache - SET last_accessed_at = CASE WHEN last_accessed_at IS NULL OR last_accessed_at < ? THEN ? ELSE last_accessed_at END - WHERE ecosystem = ? AND name = ?` - -// writtenAtUTC returns a metadata_cache updated_at or created_at as the -// instant it was written. Both were written in local time, and a Postgres -// TIMESTAMP keeps only the wall clock, which lib/pq reads back as UTC. -func (db *DB) writtenAtUTC(t sql.NullTime) sql.NullTime { - if t.Valid && db.dialect == DialectPostgres { - w := t.Time - t.Time = time.Date(w.Year(), w.Month(), w.Day(), w.Hour(), w.Minute(), w.Second(), w.Nanosecond(), time.Local) - } - if t.Valid { - t.Time = t.Time.UTC() - } - return t -} - -// BackfillMetadataLastAccessed sets last_accessed_at from the write time on -// up to limit rows that have none, in one transaction, and reports how many it -// set. Rows served in the meantime keep the time of that hit. -func (db *DB) BackfillMetadataLastAccessed(limit int) (int, error) { - var rows []struct { - ID int64 `db:"id"` - UpdatedAt sql.NullTime `db:"updated_at"` - CreatedAt sql.NullTime `db:"created_at"` - } - query := db.Rebind(`SELECT id, updated_at, created_at FROM metadata_cache WHERE last_accessed_at IS NULL LIMIT ?`) - if err := db.Select(&rows, query, limit); err != nil { - return 0, err - } - if len(rows) == 0 { - return 0, nil - } - tx, err := db.Beginx() - if err != nil { - return 0, err - } - defer func() { _ = tx.Rollback() }() - stmt, err := tx.Preparex(db.Rebind(`UPDATE metadata_cache SET last_accessed_at = ? WHERE id = ? AND last_accessed_at IS NULL`)) - if err != nil { - return 0, err - } - defer func() { _ = stmt.Close() }() - for _, r := range rows { - at := db.writtenAtUTC(r.UpdatedAt) - if !at.Valid { - at = db.writtenAtUTC(r.CreatedAt) - } - if !at.Valid { - // Nothing to go by; counting it as used now at least ends the loop. - at = sql.NullTime{Time: time.Now().UTC(), Valid: true} - } - if _, err := stmt.Exec(at.Time, r.ID); err != nil { - return 0, err - } - } - return len(rows), tx.Commit() -} diff --git a/internal/database/cache_eviction_test.go b/internal/database/cache_eviction_test.go deleted file mode 100644 index 4040316..0000000 --- a/internal/database/cache_eviction_test.go +++ /dev/null @@ -1,325 +0,0 @@ -package database - -import ( - "database/sql" - "strings" - "testing" - "time" -) - -func upsertTestMetadata(t *testing.T, db *DB, name string, size int64) { - t.Helper() - if err := db.UpsertMetadataCache(&MetadataCacheEntry{ - Ecosystem: testEcosystemNPM, - Name: name, - StoragePath: "_metadata/npm/" + name + "/metadata", - Size: sql.NullInt64{Int64: size, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - }); err != nil { - t.Fatalf("UpsertMetadataCache: %v", err) - } -} - -func setTestMetadataAccess(t *testing.T, db *DB, name string, at time.Time) { - t.Helper() - query := db.Rebind(`UPDATE metadata_cache SET last_accessed_at = ? WHERE ecosystem = ? AND name = ?`) - if _, err := db.Exec(query, at.UTC(), testEcosystemNPM, name); err != nil { - t.Fatalf("setting last access: %v", err) - } -} - -func TestUpsertMetadataCacheMarksAccess(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - before := time.Now().Add(-time.Second) - upsertTestMetadata(t, db, "lodash", 10) - - got, err := db.GetMetadataCache(testEcosystemNPM, "lodash") - if err != nil || got == nil { - t.Fatalf("GetMetadataCache: %v, %v", got, err) - } - if !got.LastAccessedAt.Valid || got.LastAccessedAt.Time.Before(before) { - t.Errorf("last_accessed_at = %v, want the time of the write", got.LastAccessedAt) - } - }) -} - -func TestGetEvictableCacheSizeCountsMetadata(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - upsertTestMetadata(t, db, "a", 300) - upsertTestMetadata(t, db, "b", 200) - - total, err := db.GetEvictableCacheSize() - if err != nil { - t.Fatalf("GetEvictableCacheSize: %v", err) - } - if total != 500 { - t.Errorf("evictable size = %d, want 500", total) - } - // The size shown in stats stays the artifacts' total. - artifacts, err := db.GetTotalCacheSize() - if err != nil || artifacts != 0 { - t.Errorf("GetTotalCacheSize = %d, %v, want 0", artifacts, err) - } - }) -} - -func TestGetLeastRecentlyUsedCacheEntriesOrdersMetadataAndArtifacts(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now() - upsertTestMetadata(t, db, "old", 1) - setTestMetadataAccess(t, db, "old", now.Add(-3*time.Hour)) - upsertTestMetadata(t, db, "new", 1) - setTestMetadataAccess(t, db, "new", now) - if err := db.UpsertArtifact(&Artifact{ - VersionPURL: "pkg:npm/mid@1.0.0", - Filename: "mid-1.0.0.tgz", - UpstreamURL: "https://example.test/mid-1.0.0.tgz", - StoragePath: sql.NullString{String: "npm/mid/1.0.0/mid-1.0.0.tgz", Valid: true}, - Size: sql.NullInt64{Int64: 1, Valid: true}, - LastAccessedAt: sql.NullTime{Time: now.Add(-time.Hour), Valid: true}, - }); err != nil { - t.Fatalf("UpsertArtifact: %v", err) - } - - entries, err := db.GetLeastRecentlyUsedCacheEntries(10) - if err != nil { - t.Fatalf("GetLeastRecentlyUsedCacheEntries: %v", err) - } - var got []string - for _, e := range entries { - got = append(got, e.Kind+":"+e.Name) - } - want := []string{"metadata:old", "artifact:mid-1.0.0.tgz", "metadata:new"} - if len(got) != len(want) { - t.Fatalf("entries = %v, want %v", got, want) - } - for i := range want { - if got[i] != want[i] { - t.Fatalf("entries = %v, want %v", got, want) - } - } - }) -} - -func TestEvictMetadataCacheKeepsRowUsedSinceRead(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - upsertTestMetadata(t, db, "lodash", 10) - setTestMetadataAccess(t, db, "lodash", time.Now().Add(-time.Hour)) - read, _ := db.GetMetadataCache(testEcosystemNPM, "lodash") - - if err := db.RecordMetadataHit(testEcosystemNPM, "lodash"); err != nil { - t.Fatalf("RecordMetadataHit: %v", err) - } - evicted, err := db.EvictMetadataCache(testEcosystemNPM, "lodash", read.StoragePath, read.LastAccessedAt) - if err != nil || evicted { - t.Fatalf("EvictMetadataCache = %v, %v, want the row kept", evicted, err) - } - - current, _ := db.GetMetadataCache(testEcosystemNPM, "lodash") - evicted, err = db.EvictMetadataCache(testEcosystemNPM, "lodash", current.StoragePath, current.LastAccessedAt) - if err != nil || !evicted { - t.Fatalf("EvictMetadataCache = %v, %v, want the row evicted", evicted, err) - } - if gone, _ := db.GetMetadataCache(testEcosystemNPM, "lodash"); gone != nil { - t.Error("row still present after eviction") - } - }) -} - -func TestRecordMetadataHitBatched(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - upsertTestMetadata(t, db, "lodash", 10) - old := time.Now().Add(-time.Hour) - setTestMetadataAccess(t, db, "lodash", old) - db.BatchHits(time.Hour, nil) - - if err := db.RecordMetadataHit(testEcosystemNPM, "lodash"); err != nil { - t.Fatalf("RecordMetadataHit: %v", err) - } - if got, _ := db.GetMetadataCache(testEcosystemNPM, "lodash"); got.LastAccessedAt.Time.After(old.Add(time.Minute)) { - t.Error("batched hit was written before the flush") - } - if err := db.flushHits(); err != nil { - t.Fatalf("flushHits: %v", err) - } - if got, _ := db.GetMetadataCache(testEcosystemNPM, "lodash"); !got.LastAccessedAt.Time.After(old.Add(time.Minute)) { - t.Errorf("last_accessed_at = %v after flush, want the hit's time", got.LastAccessedAt) - } - }) -} - -func TestPendingDeleteSkipsPathMetadataPointsAt(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - upsertTestMetadata(t, db, "lodash", 10) - path := "_metadata/npm/lodash/metadata" - if err := db.QueuePendingDelete(path); err != nil { - t.Fatalf("QueuePendingDelete: %v", err) - } - - due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 10) - if err != nil { - t.Fatalf("GetDuePendingDeletes: %v", err) - } - if len(due) != 0 { - t.Errorf("due = %v, want the path a metadata row uses skipped", due) - } - }) -} - -// TestGetLeastRecentlyUsedCacheEntriesCountsFetchAsUse checks that an -// artifact not served since its download ranks by when it was fetched, not -// ahead of every metadata document. -func TestGetLeastRecentlyUsedCacheEntriesCountsFetchAsUse(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now() - upsertTestMetadata(t, db, "old", 1) - setTestMetadataAccess(t, db, "old", now.Add(-time.Hour)) - if err := db.UpsertArtifact(&Artifact{ - VersionPURL: "pkg:npm/fresh@1.0.0", - Filename: "fresh-1.0.0.tgz", - UpstreamURL: "https://example.test/fresh-1.0.0.tgz", - StoragePath: sql.NullString{String: "npm/fresh/1.0.0/fresh-1.0.0.tgz", Valid: true}, - Size: sql.NullInt64{Int64: 1, Valid: true}, - FetchedAt: sql.NullTime{Time: now, Valid: true}, - }); err != nil { - t.Fatalf("UpsertArtifact: %v", err) - } - - entries, err := db.GetLeastRecentlyUsedCacheEntries(1) - if err != nil || len(entries) != 1 { - t.Fatalf("GetLeastRecentlyUsedCacheEntries: %v, %v", entries, err) - } - if entries[0].Kind != CacheEntryMetadata { - t.Errorf("least recently used = %+v, want the metadata used an hour ago", entries[0]) - } - }) -} - -// TestLeastRecentlyUsedQueriesUseIndexes checks on SQLite that eviction reads -// both tables off an index rather than sorting them whole. -func TestLeastRecentlyUsedQueriesUseIndexes(t *testing.T) { - db := setupMetadataCacheDB(t) - for _, q := range []string{ - `SELECT version_purl FROM artifacts WHERE storage_path IS NOT NULL AND source <> 'local' - ORDER BY COALESCE(last_accessed_at, fetched_at) ASC NULLS FIRST LIMIT 50`, - `SELECT name FROM metadata_cache ORDER BY COALESCE(last_accessed_at, updated_at) ASC NULLS FIRST LIMIT 50`, - } { - rows, err := db.Query("EXPLAIN QUERY PLAN " + q) - if err != nil { - t.Fatalf("explain: %v", err) - } - var plan []string - for rows.Next() { - var id, parent, unused int - var detail string - if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { - t.Fatalf("scan: %v", err) - } - plan = append(plan, detail) - } - _ = rows.Close() - for _, step := range plan { - if strings.Contains(step, "TEMP B-TREE") { - t.Errorf("query sorts instead of using an index: %v\n%s", plan, q) - } - } - } -} - -// insertLegacyMetadata adds a row as the proxy wrote it before -// last_accessed_at existed: write times in local time, no access time. -func insertLegacyMetadata(t *testing.T, db *DB, name string, written time.Time) { - t.Helper() - if _, err := db.Exec(db.Rebind(` - INSERT INTO metadata_cache (ecosystem, name, storage_path, size, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?)`), testEcosystemNPM, name, "_metadata/npm/"+name+"/metadata", 1, written, written); err != nil { - t.Fatalf("inserting legacy row: %v", err) - } -} - -func TestMetadataLastAccessedMigrationLeavesBackfillToBackground(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - for _, q := range []string{ - "DROP INDEX idx_metadata_last_used", - "DROP INDEX idx_metadata_storage_path", - "DROP INDEX idx_artifacts_last_used", - "ALTER TABLE metadata_cache DROP COLUMN last_accessed_at", - "DELETE FROM migrations WHERE name = '014_add_metadata_last_accessed'", - } { - if _, err := db.Exec(q); err != nil { - t.Fatalf("%s: %v", q, err) - } - } - written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.Local) - insertLegacyMetadata(t, db, "legacy", written) - - if err := db.MigrateSchema(); err != nil { - t.Fatalf("MigrateSchema: %v", err) - } - if got, _ := db.GetMetadataCache(testEcosystemNPM, "legacy"); got == nil || got.LastAccessedAt.Valid { - t.Fatalf("row after migration = %+v, want it kept with no access time", got) - } - if n, err := db.BackfillMetadataLastAccessed(10); err != nil || n != 1 { - t.Fatalf("BackfillMetadataLastAccessed = %d, %v, want 1", n, err) - } - got, _ := db.GetMetadataCache(testEcosystemNPM, "legacy") - if got == nil || !got.LastAccessedAt.Valid || !got.LastAccessedAt.Time.Equal(written) { - t.Errorf("backfilled last_accessed_at = %+v, want %v", got, written.UTC()) - } - }) -} - -// TestBackfillMetadataLastAccessedResumes runs the backfill in batches, as a -// restart after an interruption would, and leaves rows served meanwhile alone. -func TestBackfillMetadataLastAccessedResumes(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - written := time.Now().Add(-48 * time.Hour).Truncate(time.Second) - for _, name := range []string{"a", "b", "c"} { - insertLegacyMetadata(t, db, name, written) - } - if n, err := db.BackfillMetadataLastAccessed(2); err != nil || n != 2 { - t.Fatalf("first batch = %d, %v, want 2", n, err) - } - var left string - if err := db.Get(&left, `SELECT name FROM metadata_cache WHERE last_accessed_at IS NULL`); err != nil { - t.Fatalf("finding the row left: %v", err) - } - if err := db.RecordMetadataHit(testEcosystemNPM, left); err != nil { - t.Fatalf("RecordMetadataHit: %v", err) - } - if n, err := db.BackfillMetadataLastAccessed(2); err != nil || n != 0 { - t.Fatalf("second batch = %d, %v, want 0", n, err) - } - got, _ := db.GetMetadataCache(testEcosystemNPM, left) - if !got.LastAccessedAt.Time.After(written.Add(time.Hour)) { - t.Errorf("served row last_accessed_at = %v, want the hit kept", got.LastAccessedAt) - } - }) -} - -// TestLeastRecentlyUsedOrdersUnbackfilledMetadataByWriteTime checks that rows -// still waiting for the backfill are not all evicted first. -func TestLeastRecentlyUsedOrdersUnbackfilledMetadataByWriteTime(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now() - insertLegacyMetadata(t, db, "recent", now) - if err := db.UpsertArtifact(&Artifact{ - VersionPURL: "pkg:npm/old@1.0.0", - Filename: "old-1.0.0.tgz", - UpstreamURL: "https://example.test/old-1.0.0.tgz", - StoragePath: sql.NullString{String: "npm/old/1.0.0/old-1.0.0.tgz", Valid: true}, - Size: sql.NullInt64{Int64: 1, Valid: true}, - LastAccessedAt: sql.NullTime{Time: now.Add(-time.Hour), Valid: true}, - }); err != nil { - t.Fatalf("UpsertArtifact: %v", err) - } - - entries, err := db.GetLeastRecentlyUsedCacheEntries(1) - if err != nil || len(entries) != 1 { - t.Fatalf("GetLeastRecentlyUsedCacheEntries: %v, %v", entries, err) - } - if entries[0].Kind != CacheEntryArtifact { - t.Errorf("least recently used = %+v, want the artifact used an hour ago", entries[0]) - } - }) -} diff --git a/internal/database/cached_names.go b/internal/database/cached_names.go deleted file mode 100644 index 0b9563b..0000000 --- a/internal/database/cached_names.go +++ /dev/null @@ -1,21 +0,0 @@ -package database - -// CachedArtifactNames returns the names of the packages of ecosystem that -// have a cached artifact called filename, for protocols whose file names -// identify an artifact without its package name. -func (db *DB) CachedArtifactNames(ecosystem, filename string) ([]string, error) { - var names []string - query := db.Rebind(` - SELECT DISTINCT packages.name - FROM artifacts - JOIN versions ON versions.purl = artifacts.version_purl - JOIN packages ON packages.purl = versions.package_purl - WHERE packages.ecosystem = ? AND artifacts.filename = ? - AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL - ORDER BY packages.name - `) - if err := db.Select(&names, query, ecosystem, filename); err != nil { - return nil, err - } - return names, nil -} diff --git a/internal/database/database.go b/internal/database/database.go index ac71ac1..eded6d2 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,14 +1,9 @@ package database import ( - "context" - "database/sql" "fmt" "os" "path/filepath" - "runtime" - "strings" - "time" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" @@ -19,16 +14,6 @@ const SchemaVersion = 1 const dirPermissions = 0755 -// Postgres connection pool limits. database/sql keeps only two idle -// connections by default, which opens a new Postgres session for almost every -// request under load. -const ( - postgresMaxOpenConns = 32 - postgresMaxIdleConns = 32 - postgresConnMaxIdleTime = 5 * time.Minute - postgresConnMaxLifetime = 30 * time.Minute -) - type Dialect string const ( @@ -36,36 +21,10 @@ const ( DialectPostgres Dialect = "postgres" ) -// SQLite DSN parameters. Every connection a pool opens applies them, so a -// connection database/sql reopens gets them too. The reader pool sets -// query_only, which makes a write sent to it fail instead of being run. -const ( - sqliteWriterParams = "?_busy_timeout=5000&_journal_mode=WAL&_synchronous=NORMAL" - sqliteReaderParams = "?_busy_timeout=5000&_query_only=1" -) - -// Size bounds of the SQLite reader pool. At least two, so a slow stats or -// eviction scan does not block request lookups even on one CPU. -const ( - sqliteMinReaders = 2 - sqliteMaxReaders = 8 -) - -// DB is the proxy database. The embedded pool takes every write. For SQLite -// it holds a single connection, as SQLite allows one writer at a time, and -// reads go to a separate read-only pool so they don't queue behind writes or -// behind each other (WAL mode lets readers run alongside the writer). For -// Postgres and in-memory SQLite both are the same pool. -// -// Get, Select, Query and QueryRow (and their Context forms) use the reader. -// Exec, transactions and everything else use the writer, so a read inside a -// transaction goes through the transaction and sees its own writes. type DB struct { *sqlx.DB - reader *sqlx.DB dialect Dialect path string - hits *hitBatch } func (db *DB) Dialect() Dialect { @@ -104,51 +63,24 @@ func Open(path string) (*DB, error) { } } - writer, err := sqlx.Open("sqlite", path+sqliteWriterParams) + // Add busy_timeout to handle concurrent writes + sqlDB, err := sqlx.Open("sqlite", path+"?_busy_timeout=5000") if err != nil { return nil, fmt.Errorf("opening database: %w", err) } - // One connection serializes writes, so they never wait on SQLite's - // busy timeout for each other. - writer.SetMaxOpenConns(1) - // Connecting applies journal_mode=WAL before any reader opens the file. - if err := writer.Ping(); err != nil { - _ = writer.Close() - return nil, fmt.Errorf("opening database: %w", err) - } + // Limit connections to 1 for SQLite to serialize writes + sqlDB.SetMaxOpenConns(1) - db := &DB{DB: writer, reader: writer, dialect: DialectSQLite, path: path} - if isMemoryPath(path) { - // Each connection to an in-memory database gets its own empty - // database, so the reads have to share the writer's connection. - return db, nil + db := &DB{DB: sqlDB, dialect: DialectSQLite, path: path} + if err := db.OptimizeForReads(); err != nil { + _ = sqlDB.Close() + return nil, fmt.Errorf("optimizing database: %w", err) } - reader, err := sqlx.Open("sqlite", path+sqliteReaderParams) - if err != nil { - _ = writer.Close() - return nil, fmt.Errorf("opening database reader: %w", err) - } - n := sqliteReaderConns() - reader.SetMaxOpenConns(n) - reader.SetMaxIdleConns(n) - db.reader = reader - return db, nil } -func sqliteReaderConns() int { - return min(max(runtime.GOMAXPROCS(0), sqliteMinReaders), sqliteMaxReaders) -} - -// isMemoryPath reports whether path names an in-memory database, as -// ":memory:" or a URI such as "file::memory:?cache=shared" or -// "file:x?mode=memory". -func isMemoryPath(path string) bool { - return path == "" || strings.Contains(path, ":memory:") || strings.Contains(path, "mode=memory") -} - func OpenOrCreate(path string) (*DB, error) { if Exists(path) { return Open(path) @@ -162,17 +94,12 @@ func OpenPostgres(url string) (*DB, error) { return nil, fmt.Errorf("opening postgres database: %w", err) } - sqlDB.SetMaxOpenConns(postgresMaxOpenConns) - sqlDB.SetMaxIdleConns(postgresMaxIdleConns) - sqlDB.SetConnMaxIdleTime(postgresConnMaxIdleTime) - sqlDB.SetConnMaxLifetime(postgresConnMaxLifetime) - if err := sqlDB.Ping(); err != nil { _ = sqlDB.Close() return nil, fmt.Errorf("connecting to postgres: %w", err) } - return &DB{DB: sqlDB, reader: sqlDB, dialect: DialectPostgres}, nil + return &DB{DB: sqlDB, dialect: DialectPostgres}, nil } func OpenPostgresOrCreate(url string) (*DB, error) { @@ -181,24 +108,18 @@ func OpenPostgresOrCreate(url string) (*DB, error) { return nil, err } - // Instances started together on an empty database would otherwise - // create the schema at the same time and fail on each other's tables. - err = db.WithLock(context.Background(), schemaLock, func() error { - var exists bool - if err := db.Get(&exists, "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'schema_info')"); err != nil { - return fmt.Errorf("checking schema: %w", err) - } - if exists { - return nil - } - if err := db.CreateSchema(); err != nil { - return fmt.Errorf("creating schema: %w", err) - } - return nil - }) + var exists bool + err = db.Get(&exists, "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'schema_info')") if err != nil { _ = db.Close() - return nil, err + return nil, fmt.Errorf("checking schema: %w", err) + } + + if !exists { + if err := db.CreateSchema(); err != nil { + _ = db.Close() + return nil, fmt.Errorf("creating schema: %w", err) + } } return db, nil @@ -234,56 +155,3 @@ func (db *DB) Path() string { func (db *DB) Rebind(query string) string { return db.DB.Rebind(query) } - -// Get runs a read on the reader pool. -func (db *DB) Get(dest any, query string, args ...any) error { - return db.reader.Get(dest, query, args...) -} - -// GetContext runs a read on the reader pool. -func (db *DB) GetContext(ctx context.Context, dest any, query string, args ...any) error { - return db.reader.GetContext(ctx, dest, query, args...) -} - -// Select runs a read on the reader pool. -func (db *DB) Select(dest any, query string, args ...any) error { - return db.reader.Select(dest, query, args...) -} - -// SelectContext runs a read on the reader pool. -func (db *DB) SelectContext(ctx context.Context, dest any, query string, args ...any) error { - return db.reader.SelectContext(ctx, dest, query, args...) -} - -// Query runs a read on the reader pool. -func (db *DB) Query(query string, args ...any) (*sql.Rows, error) { - return db.reader.Query(query, args...) -} - -// QueryContext runs a read on the reader pool. -func (db *DB) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { - return db.reader.QueryContext(ctx, query, args...) -} - -// QueryRow runs a read on the reader pool. -func (db *DB) QueryRow(query string, args ...any) *sql.Row { - return db.reader.QueryRow(query, args...) -} - -// QueryRowContext runs a read on the reader pool. -func (db *DB) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row { - return db.reader.QueryRowContext(ctx, query, args...) -} - -// closePools closes the reader pool, if separate, then the writer. The -// writer goes last so that, as the last connection, it checkpoints the WAL. -func (db *DB) closePools() error { - var readerErr error - if db.reader != nil && db.reader != db.DB { - readerErr = db.reader.Close() - } - if err := db.DB.Close(); err != nil { - return err - } - return readerErr -} diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 500ad0d..6fca4ea 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -8,11 +8,6 @@ import ( "time" ) -const ( - testContentHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - testIntegrity = "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" -) - func TestCreateAndOpen(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") @@ -137,7 +132,7 @@ func TestVersionCRUD(t *testing.T) { v := &Version{ PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash", - Integrity: sql.NullString{String: testIntegrity, Valid: true}, + Integrity: sql.NullString{String: "sha512-abc123", Valid: true}, } err = db.UpsertVersion(v) @@ -205,7 +200,7 @@ func TestArtifactCRUD(t *testing.T) { t.Error("expected artifact to not be cached yet") } - err = db.MarkArtifactCached(versionPURL, "lodash-4.17.21.tgz", "/cache/npm/lodash-4.17.21.tgz", testContentHash, 12345, "application/gzip") + err = db.MarkArtifactCached(versionPURL, "lodash-4.17.21.tgz", "/cache/npm/lodash-4.17.21.tgz", "sha256-abc", 12345, "application/gzip") if err != nil { t.Fatalf("MarkArtifactCached failed: %v", err) } @@ -244,92 +239,6 @@ func TestArtifactCRUD(t *testing.T) { }) } -func TestGetCachedArtifact(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - const ( - packagePURL = "pkg:npm/lodash" - versionPURL = "pkg:npm/lodash@4.17.21" - filename = "lodash-4.17.21.tgz" - ) - seedCachedArtifactTestData(t, db, packagePURL, versionPURL, filename) - - cached, err := db.GetCachedArtifact(packagePURL, versionPURL, filename) - if err != nil { - t.Fatalf("GetCachedArtifact before cache failed: %v", err) - } - if cached != nil { - t.Fatalf("expected no cached artifact, got %+v", cached) - } - - if err := db.MarkArtifactCached(versionPURL, filename, "/cache/npm/"+filename, - testContentHash, 12345, "application/gzip"); err != nil { - t.Fatalf("MarkArtifactCached failed: %v", err) - } - - cached, err = db.GetCachedArtifact(packagePURL, versionPURL, filename) - if err != nil { - t.Fatalf("GetCachedArtifact failed: %v", err) - } - if cached == nil { - t.Fatal("expected cached artifact, got nil") - } - if cached.Ecosystem != "npm" { - t.Errorf("expected npm ecosystem, got %q", cached.Ecosystem) - } - if cached.StoragePath != "/cache/npm/"+filename { - t.Errorf("expected cached storage path, got %q", cached.StoragePath) - } - if cached.Artifact.PURL != versionPURL { - t.Errorf("expected cached PURL %q, got %q", versionPURL, cached.Artifact.PURL) - } - if cached.Artifact.Digest.String() != "sha256:"+testContentHash { - t.Errorf("expected cached digest, got %q", cached.Artifact.Digest) - } - if cached.Artifact.Size != 12345 { - t.Errorf("expected cached size 12345, got %d", cached.Artifact.Size) - } - if cached.Artifact.Filename != filename { - t.Errorf("expected cached filename %q, got %q", filename, cached.Artifact.Filename) - } - if cached.Artifact.MediaType != "application/gzip" { - t.Errorf("expected cached content type, got %q", cached.Artifact.MediaType) - } - if cached.Integrity.String != testIntegrity { - t.Errorf("expected cached integrity, got %q", cached.Integrity.String) - } - - cached, err = db.GetCachedArtifact("pkg:npm/other", versionPURL, filename) - if err != nil { - t.Fatalf("GetCachedArtifact with wrong package failed: %v", err) - } - if cached != nil { - t.Fatalf("expected package mismatch to miss cache, got %+v", cached) - } - }) -} - -func seedCachedArtifactTestData(t *testing.T, db *DB, packagePURL, versionPURL, filename string) { - t.Helper() - - if err := db.UpsertPackage(&Package{PURL: packagePURL, Ecosystem: "npm", Name: "lodash"}); err != nil { - t.Fatalf("UpsertPackage failed: %v", err) - } - if err := db.UpsertVersion(&Version{ - PURL: versionPURL, - PackagePURL: packagePURL, - Integrity: sql.NullString{String: testIntegrity, Valid: true}, - }); err != nil { - t.Fatalf("UpsertVersion failed: %v", err) - } - if err := db.UpsertArtifact(&Artifact{ - VersionPURL: versionPURL, - Filename: filename, - UpstreamURL: "https://registry.npmjs.org/lodash/-/" + filename, - }); err != nil { - t.Fatalf("UpsertArtifact failed: %v", err) - } -} - func TestCacheManagement(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { pkg := &Package{ @@ -610,10 +519,8 @@ func createTestPostgresDB(t *testing.T) *DB { t.Fatalf("OpenPostgres failed: %v", err) } - // Drop and recreate every table CreateSchema creates for clean test state; - // leftover migration records make the next CreateSchema fail on the - // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "metadata_snapshots", "projects", "project_components", "migrations", "schema_info"} + // Drop and recreate tables for clean test state + tables := []string{"artifacts", "versions", "packages", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } @@ -781,10 +688,6 @@ func TestMigrationFromOldSchema(t *testing.T) { t.Errorf("expected package name test-package, got %s", pkg.Name) } - if has, err := db.HasTable("pending_deletes"); err != nil || !has { - t.Errorf("pending_deletes table missing after migration (err %v)", err) - } - // Verify migrations were recorded applied, err := db.appliedMigrations() if err != nil { @@ -1111,139 +1014,3 @@ func BenchmarkMigrateSchemaFullyMigrated(b *testing.B) { } } } - -func TestVersionPublishedAtPreserved(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - publishedAt := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) - - if err := db.SetVersionPublishedAt("pkg:npm/leftpad@1.0.0", "pkg:npm/leftpad", publishedAt); err != nil { - t.Fatalf("SetVersionPublishedAt failed: %v", err) - } - - got, err := db.GetVersionByPURL("pkg:npm/leftpad@1.0.0") - if err != nil || got == nil { - t.Fatalf("GetVersionByPURL failed: %v", err) - } - if !got.PublishedAt.Valid || !got.PublishedAt.Time.Equal(publishedAt) { - t.Fatalf("PublishedAt = %v, want %v", got.PublishedAt, publishedAt) - } - - // An upsert that carries no publish time (the artifact cache path) - // must not erase the stored value. - if err := db.UpsertVersion(&Version{ - PURL: "pkg:npm/leftpad@1.0.0", - PackagePURL: "pkg:npm/leftpad", - }); err != nil { - t.Fatalf("UpsertVersion failed: %v", err) - } - - got, err = db.GetVersionByPURL("pkg:npm/leftpad@1.0.0") - if err != nil || got == nil { - t.Fatalf("GetVersionByPURL after upsert failed: %v", err) - } - if !got.PublishedAt.Valid || !got.PublishedAt.Time.Equal(publishedAt) { - t.Fatalf("PublishedAt after null upsert = %v, want %v preserved", got.PublishedAt, publishedAt) - } - - // An upsert that does carry a publish time still updates it. - later := publishedAt.Add(24 * time.Hour) - if err := db.UpsertVersion(&Version{ - PURL: "pkg:npm/leftpad@1.0.0", - PackagePURL: "pkg:npm/leftpad", - PublishedAt: sql.NullTime{Time: later, Valid: true}, - }); err != nil { - t.Fatalf("UpsertVersion with publish time failed: %v", err) - } - - got, err = db.GetVersionByPURL("pkg:npm/leftpad@1.0.0") - if err != nil || got == nil { - t.Fatalf("GetVersionByPURL after second upsert failed: %v", err) - } - if !got.PublishedAt.Valid || !got.PublishedAt.Time.Equal(later) { - t.Fatalf("PublishedAt after valued upsert = %v, want %v", got.PublishedAt, later) - } - }) -} - -func TestGetFetchedVersionPURLs(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - const pkg = "pkg:npm/leftpad" - for _, v := range []string{"1.0.0", "2.0.0", "3.0.0"} { - if err := db.UpsertVersion(&Version{PURL: pkg + "@" + v, PackagePURL: pkg}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - } - if err := db.UpsertVersion(&Version{PURL: "pkg:npm/other@1.0.0", PackagePURL: "pkg:npm/other"}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - artifacts := []Artifact{ - {VersionPURL: pkg + "@1.0.0", Filename: "leftpad-1.0.0.tgz", UpstreamURL: "https://example.com/1", - StoragePath: sql.NullString{String: "npm/leftpad-1.0.0.tgz", Valid: true}}, - // Evicted: the row stays without a storage path. - {VersionPURL: pkg + "@2.0.0", Filename: "leftpad-2.0.0.tgz", UpstreamURL: "https://example.com/2"}, - {VersionPURL: "pkg:npm/other@1.0.0", Filename: "other-1.0.0.tgz", UpstreamURL: "https://example.com/3"}, - } - for i := range artifacts { - if err := db.UpsertArtifact(&artifacts[i]); err != nil { - t.Fatalf("UpsertArtifact: %v", err) - } - } - - got, err := db.GetFetchedVersionPURLs(pkg) - if err != nil { - t.Fatalf("GetFetchedVersionPURLs: %v", err) - } - want := map[string]bool{pkg + "@1.0.0": true, pkg + "@2.0.0": true} - if len(got) != len(want) { - t.Fatalf("got %v, want %v", got, want) - } - for purl := range want { - if !got[purl] { - t.Errorf("missing %s in %v", purl, got) - } - } - }) -} - -// Publish times are written in UTC whatever zone the caller's time is in, so -// a TIMESTAMP column without a zone reads back the same instant. -func TestVersionPublishedAtIsUTC(t *testing.T) { - tokyo, err := time.LoadLocation("Asia/Tokyo") - if err != nil { - t.Skipf("time zone data unavailable: %v", err) - } - prev := time.Local - time.Local = tokyo - t.Cleanup(func() { time.Local = prev }) - - runWithBothDatabases(t, func(t *testing.T, db *DB) { - written := time.Date(2026, 10, 10, 21, 0, 1, 0, tokyo) - want := written.UTC() - check := func(purl string) { - t.Helper() - got, err := db.GetVersionByPURL(purl) - if err != nil || got == nil || !got.PublishedAt.Valid { - t.Fatalf("GetVersionByPURL(%s) = %+v, %v", purl, got, err) - } - // Postgres reads TIMESTAMP back as a fixed +0000 zone rather - // than time.UTC, so compare the offset. - if _, offset := got.PublishedAt.Time.Zone(); !got.PublishedAt.Time.Equal(want) || offset != 0 { - t.Errorf("%s PublishedAt read back as %v, want %v", purl, got.PublishedAt.Time, want) - } - } - - if err := db.SetVersionPublishedAt("pkg:cargo/demo@1.0.0", "pkg:cargo/demo", written); err != nil { - t.Fatal(err) - } - check("pkg:cargo/demo@1.0.0") - - if err := db.UpsertVersion(&Version{ - PURL: "pkg:cargo/demo@2.0.0", - PackagePURL: "pkg:cargo/demo", - PublishedAt: sql.NullTime{Time: written, Valid: true}, - }); err != nil { - t.Fatal(err) - } - check("pkg:cargo/demo@2.0.0") - }) -} diff --git a/internal/database/ecosystem_filter.go b/internal/database/ecosystem_filter.go deleted file mode 100644 index 17d66ba..0000000 --- a/internal/database/ecosystem_filter.go +++ /dev/null @@ -1,34 +0,0 @@ -package database - -import ( - "slices" - "strings" - - "github.com/git-pkgs/purl" -) - -// ecosystemSpellings returns the names an ecosystem's packages can be stored -// under. The proxy's handlers write "gem" and "composer" where git-pkgs, and -// proxy mirror before it used the handler names, write "rubygems" and -// "packagist". -func ecosystemSpellings(ecosystem string) []string { - purlType := purl.EcosystemToPURLType(ecosystem) - spellings := []string{ecosystem} - for _, s := range []string{purlType, purl.PURLTypeToEcosystem(purlType)} { - if s != "" && !slices.Contains(spellings, s) { - spellings = append(spellings, s) - } - } - return spellings -} - -// ecosystemFilter returns a condition matching p.ecosystem against every -// spelling of ecosystem, and its arguments. -func ecosystemFilter(ecosystem string) (string, []any) { - spellings := ecosystemSpellings(ecosystem) - args := make([]any, len(spellings)) - for i, s := range spellings { - args[i] = s - } - return "p.ecosystem IN (" + strings.TrimSuffix(strings.Repeat("?, ", len(spellings)), ", ") + ")", args -} diff --git a/internal/database/fetch_leases.go b/internal/database/fetch_leases.go deleted file mode 100644 index 56390e3..0000000 --- a/internal/database/fetch_leases.go +++ /dev/null @@ -1,94 +0,0 @@ -package database - -import ( - "database/sql" - "errors" - "fmt" - "time" -) - -// Fetch leases let instances sharing a Postgres database fetch each cache -// miss from upstream once. The instance that fetches holds the lease for the -// miss's key and renews it until the result is stored; the others wait for -// the stored result. Expiry uses the database clock, so instances whose -// clocks disagree still agree on when a lease has run out. SQLite is -// single-instance, so these are only used on Postgres. - -// AcquireFetchLease takes the lease on key for owner unless another owner -// holds one that has neither expired nor failed, and reports whether it did. -func (db *DB) AcquireFetchLease(key, owner string, ttl time.Duration) (bool, error) { - res, err := db.Exec(` - INSERT INTO fetch_leases (key, owner, expires_at) - VALUES ($1, $2, now() + $3 * interval '1 millisecond') - ON CONFLICT (key) DO UPDATE SET owner = excluded.owner, expires_at = excluded.expires_at, failed = FALSE - WHERE fetch_leases.expires_at < now() OR fetch_leases.failed`, key, owner, ttl.Milliseconds()) - if err != nil { - return false, fmt.Errorf("acquiring fetch lease: %w", err) - } - n, err := res.RowsAffected() - if err != nil { - return false, fmt.Errorf("acquiring fetch lease: %w", err) - } - return n > 0, nil -} - -// RenewFetchLease extends owner's lease on key by ttl from now, and reports -// whether owner still held it. -func (db *DB) RenewFetchLease(key, owner string, ttl time.Duration) (bool, error) { - res, err := db.Exec(` - UPDATE fetch_leases SET expires_at = now() + $3 * interval '1 millisecond' - WHERE key = $1 AND owner = $2`, key, owner, ttl.Milliseconds()) - if err != nil { - return false, fmt.Errorf("renewing fetch lease: %w", err) - } - n, err := res.RowsAffected() - if err != nil { - return false, fmt.Errorf("renewing fetch lease: %w", err) - } - return n > 0, nil -} - -// FetchLeaseHolder returns the owner of the lease on key, and whether its -// fetch failed. ok is false when no lease is held or it has expired. -func (db *DB) FetchLeaseHolder(key string) (owner string, failed, ok bool, err error) { - err = db.QueryRow(` - SELECT owner, failed FROM fetch_leases - WHERE key = $1 AND expires_at >= now()`, key).Scan(&owner, &failed) - if errors.Is(err, sql.ErrNoRows) { - return "", false, false, nil - } - if err != nil { - return "", false, false, fmt.Errorf("reading fetch lease: %w", err) - } - return owner, failed, true, nil -} - -// FailFetchLease marks owner's lease on key as failed. Instances waiting on -// it stop waiting, and the next AcquireFetchLease takes it at once. -func (db *DB) FailFetchLease(key, owner string) error { - if _, err := db.Exec(`UPDATE fetch_leases SET failed = TRUE WHERE key = $1 AND owner = $2`, key, owner); err != nil { - return fmt.Errorf("failing fetch lease: %w", err) - } - return nil -} - -// ReleaseFetchLease drops owner's lease on key. -func (db *DB) ReleaseFetchLease(key, owner string) error { - if _, err := db.Exec(`DELETE FROM fetch_leases WHERE key = $1 AND owner = $2`, key, owner); err != nil { - return fmt.Errorf("releasing fetch lease: %w", err) - } - return nil -} - -// DeleteExpiredFetchLeases removes leases that expired more than grace ago, -// left by instances that stopped while fetching. -func (db *DB) DeleteExpiredFetchLeases(grace time.Duration) (int64, error) { - if db.dialect != DialectPostgres { - return 0, nil - } - res, err := db.Exec(`DELETE FROM fetch_leases WHERE expires_at < now() - $1 * interval '1 millisecond'`, grace.Milliseconds()) - if err != nil { - return 0, fmt.Errorf("deleting expired fetch leases: %w", err) - } - return res.RowsAffected() -} diff --git a/internal/database/hits.go b/internal/database/hits.go deleted file mode 100644 index 3609b5f..0000000 --- a/internal/database/hits.go +++ /dev/null @@ -1,215 +0,0 @@ -package database - -import ( - "cmp" - "errors" - "log/slog" - "slices" - "sync" - "time" -) - -// Recording each cache hit as its own UPDATE makes every cached download a -// write transaction, and SQLite runs on a single connection, so concurrent -// downloads queue behind those writes. BatchHits counts hits in memory and -// writes them in one transaction per interval instead. - -type hitKey struct{ versionPURL, filename string } - -type hitEntry struct { - count int64 - last time.Time -} - -// metadataHitKey identifies a metadata cache row; a hit only moves its -// last-access time. -type metadataHitKey struct{ ecosystem, name string } - -type hitBatch struct { - mu sync.Mutex - pending map[hitKey]hitEntry - metadata map[metadataHitKey]time.Time - stop chan struct{} - stopOnce sync.Once - done chan struct{} -} - -func (b *hitBatch) add(k hitKey, e hitEntry) { - b.mu.Lock() - defer b.mu.Unlock() - cur := b.pending[k] - cur.count += e.count - if e.last.After(cur.last) { - cur.last = e.last - } - b.pending[k] = cur -} - -func (b *hitBatch) addMetadata(k metadataHitKey, last time.Time) { - b.mu.Lock() - defer b.mu.Unlock() - if last.After(b.metadata[k]) { - b.metadata[k] = last - } -} - -func (b *hitBatch) takeMetadata() map[metadataHitKey]time.Time { - b.mu.Lock() - defer b.mu.Unlock() - metadata := b.metadata - b.metadata = map[metadataHitKey]time.Time{} - return metadata -} - -func (b *hitBatch) take() map[hitKey]hitEntry { - b.mu.Lock() - defer b.mu.Unlock() - pending := b.pending - b.pending = map[hitKey]hitEntry{} - return pending -} - -// BatchHits makes RecordArtifactHit buffer hits and write them every -// interval. An interval of zero or less leaves every hit written immediately. -// Close writes whatever is still pending. Failed writes are logged to logger. -func (db *DB) BatchHits(interval time.Duration, logger *slog.Logger) { - if interval <= 0 || db.hits != nil { - return - } - if logger == nil { - logger = slog.New(slog.DiscardHandler) - } - b := &hitBatch{pending: map[hitKey]hitEntry{}, metadata: map[metadataHitKey]time.Time{}, stop: make(chan struct{}), done: make(chan struct{})} - db.hits = b - go func() { - defer close(b.done) - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - if err := db.flushHits(); err != nil { - logger.Warn("failed to write cache hits, retrying next flush", "error", err) - } - case <-b.stop: - if err := db.flushHits(); err != nil { - logger.Error("failed to write cache hits on close, dropping them", "error", err) - } - return - } - } - }() -} - -// flushHits writes the pending hits in one transaction. If the transaction -// fails they are put back for the next flush, so a failed write delays -// counts rather than losing them. -func (db *DB) flushHits() error { - metadataErr := db.flushMetadataHits() - pending := db.hits.take() - if len(pending) == 0 { - return metadataErr - } - - err := db.writeHits(pending) - if err != nil { - for k, e := range pending { - db.hits.add(k, e) - } - } - return errors.Join(err, metadataErr) -} - -// flushMetadataHits writes the pending metadata hits like flushHits. -func (db *DB) flushMetadataHits() error { - metadata := db.hits.takeMetadata() - if len(metadata) == 0 { - return nil - } - err := db.writeMetadataHits(metadata) - if err != nil { - for k, last := range metadata { - db.hits.addMetadata(k, last) - } - } - return err -} - -func (db *DB) writeMetadataHits(metadata map[metadataHitKey]time.Time) error { - tx, err := db.Beginx() - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - stmt, err := tx.Preparex(db.Rebind(recordMetadataHitQuery)) - if err != nil { - return err - } - defer func() { _ = stmt.Close() }() - - // Sorted for the same reason as the artifact hits. - keys := make([]metadataHitKey, 0, len(metadata)) - for k := range metadata { - keys = append(keys, k) - } - slices.SortFunc(keys, func(a, b metadataHitKey) int { - return cmp.Or(cmp.Compare(a.ecosystem, b.ecosystem), cmp.Compare(a.name, b.name)) - }) - for _, k := range keys { - last := metadata[k].UTC() - if _, err := stmt.Exec(last, last, k.ecosystem, k.name); err != nil { - return err - } - } - return tx.Commit() -} - -func (db *DB) writeHits(pending map[hitKey]hitEntry) error { - tx, err := db.Beginx() - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - // Timestamps only move forward: with proxies sharing a database, an older - // batch can flush after a newer hit is already written. - stmt, err := tx.Preparex(db.Rebind(` - UPDATE artifacts - SET hit_count = hit_count + ?, - last_accessed_at = CASE WHEN last_accessed_at IS NULL OR last_accessed_at < ? THEN ? ELSE last_accessed_at END, - updated_at = CASE WHEN updated_at IS NULL OR updated_at < ? THEN ? ELSE updated_at END - WHERE version_purl = ? AND filename = ? - `)) - if err != nil { - return err - } - defer func() { _ = stmt.Close() }() - - // A fixed order keeps proxies sharing a Postgres database from locking - // the same rows in opposite orders and deadlocking. - keys := make([]hitKey, 0, len(pending)) - for k := range pending { - keys = append(keys, k) - } - slices.SortFunc(keys, func(a, b hitKey) int { - return cmp.Or(cmp.Compare(a.versionPURL, b.versionPURL), cmp.Compare(a.filename, b.filename)) - }) - for _, k := range keys { - e := pending[k] - last := e.last.UTC() - if _, err := stmt.Exec(e.count, last, last, e.last, e.last, k.versionPURL, k.filename); err != nil { - return err - } - } - return tx.Commit() -} - -// Close writes any batched hits, then closes the database. -func (db *DB) Close() error { - if b := db.hits; b != nil { - b.stopOnce.Do(func() { close(b.stop) }) - <-b.done - } - return db.closePools() -} diff --git a/internal/database/hits_test.go b/internal/database/hits_test.go deleted file mode 100644 index 99126f6..0000000 --- a/internal/database/hits_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package database - -import ( - "bytes" - "log/slog" - "path/filepath" - "strings" - "sync" - "testing" - "time" -) - -var discardLogger = slog.New(slog.DiscardHandler) - -func seedHitTestArtifact(t *testing.T, db *DB) (string, string) { - t.Helper() - versionPURL, filename := "pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz" - if err := db.UpsertPackage(&Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"}); err != nil { - t.Fatalf("UpsertPackage failed: %v", err) - } - if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: "pkg:npm/lodash"}); err != nil { - t.Fatalf("UpsertVersion failed: %v", err) - } - if err := db.UpsertArtifact(&Artifact{ - VersionPURL: versionPURL, - Filename: filename, - UpstreamURL: "https://registry.npmjs.org/lodash/-/" + filename, - }); err != nil { - t.Fatalf("UpsertArtifact failed: %v", err) - } - return versionPURL, filename -} - -func hitCount(t *testing.T, db *DB, versionPURL, filename string) int64 { - t.Helper() - a, err := db.GetArtifact(versionPURL, filename) - if err != nil || a == nil { - t.Fatalf("GetArtifact failed: %v", err) - } - return a.HitCount -} - -func TestBatchHitsWritesOnFlush(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - versionPURL, filename := seedHitTestArtifact(t, db) - db.BatchHits(time.Hour, discardLogger) - - for range 3 { - if err := db.RecordArtifactHit(versionPURL, filename); err != nil { - t.Fatalf("RecordArtifactHit failed: %v", err) - } - } - if got := hitCount(t, db, versionPURL, filename); got != 0 { - t.Fatalf("hit count before flush = %d, want 0", got) - } - - if err := db.flushHits(); err != nil { - t.Fatalf("flushHits failed: %v", err) - } - a, err := db.GetArtifact(versionPURL, filename) - if err != nil { - t.Fatalf("GetArtifact failed: %v", err) - } - if a.HitCount != 3 { - t.Errorf("hit count after flush = %d, want 3", a.HitCount) - } - if !a.LastAccessedAt.Valid { - t.Error("expected last_accessed_at to be set") - } - }) -} - -func TestBatchHitsWritesOnClose(t *testing.T) { - path := filepath.Join(t.TempDir(), "test.db") - db, err := Create(path) - if err != nil { - t.Fatalf("Create failed: %v", err) - } - versionPURL, filename := seedHitTestArtifact(t, db) - db.BatchHits(time.Hour, discardLogger) - - for range 2 { - _ = db.RecordArtifactHit(versionPURL, filename) - } - if err := db.Close(); err != nil { - t.Fatalf("Close failed: %v", err) - } - - db, err = Open(path) - if err != nil { - t.Fatalf("Open failed: %v", err) - } - defer func() { _ = db.Close() }() - if got := hitCount(t, db, versionPURL, filename); got != 2 { - t.Errorf("hit count after Close = %d, want 2", got) - } -} - -// TestBatchHitsKeepNewerAccessTime flushes a batch older than a hit already -// written, as when proxies share a database. The count still adds up, but -// neither timestamp may move backwards. -func TestBatchHitsKeepNewerAccessTime(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - versionPURL, filename := seedHitTestArtifact(t, db) - k := hitKey{versionPURL, filename} - now := time.Now() - read := func() *Artifact { - t.Helper() - a, err := db.GetArtifact(versionPURL, filename) - if err != nil || a == nil { - t.Fatalf("GetArtifact failed: %v", err) - } - return a - } - write := func(last time.Time) { - t.Helper() - if err := db.writeHits(map[hitKey]hitEntry{k: {count: 1, last: last}}); err != nil { - t.Fatalf("writeHits failed: %v", err) - } - } - - write(now) - newer := read() - if !newer.LastAccessedAt.Valid { - t.Fatal("last_accessed_at not set from NULL") - } - - write(now.Add(-time.Minute)) - got := read() - if got.HitCount != 2 { - t.Errorf("hit count = %d, want 2", got.HitCount) - } - if !got.LastAccessedAt.Time.Equal(newer.LastAccessedAt.Time) { - t.Errorf("last_accessed_at moved from %v to %v", newer.LastAccessedAt.Time, got.LastAccessedAt.Time) - } - if !got.UpdatedAt.Equal(newer.UpdatedAt) { - t.Errorf("updated_at moved from %v to %v", newer.UpdatedAt, got.UpdatedAt) - } - - write(now.Add(time.Minute)) - got = read() - if !got.LastAccessedAt.Time.After(newer.LastAccessedAt.Time) || !got.UpdatedAt.After(newer.UpdatedAt) { - t.Error("a newer batch did not advance the timestamps") - } - }) -} - -func TestBatchHitsZeroIntervalWritesImmediately(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - versionPURL, filename := seedHitTestArtifact(t, db) - db.BatchHits(0, discardLogger) - - _ = db.RecordArtifactHit(versionPURL, filename) - if got := hitCount(t, db, versionPURL, filename); got != 1 { - t.Errorf("hit count = %d, want 1", got) - } - }) -} - -func TestBatchHitsKeepsHitsWhenWriteFails(t *testing.T) { - db := createTestDB(t) - versionPURL, filename := seedHitTestArtifact(t, db) - db.BatchHits(time.Hour, discardLogger) - - for range 2 { - _ = db.RecordArtifactHit(versionPURL, filename) - } - _ = db.DB.Close() - if err := db.flushHits(); err == nil { - t.Fatal("expected flushHits to fail on a closed database") - } - if got := db.hits.take()[hitKey{versionPURL, filename}].count; got != 2 { - t.Errorf("pending hits after failed flush = %d, want 2", got) - } - _ = db.Close() -} - -// TestBatchHitsCountsEveryHitWhileFlushing records hits from several goroutines -// while another flushes, so no hit may be lost between taking the pending -// hits and writing them. -func TestBatchHitsCountsEveryHitWhileFlushing(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - versionPURL, filename := seedHitTestArtifact(t, db) - db.BatchHits(time.Hour, discardLogger) - - const writers, hitsEach = 8, 100 - stop := make(chan struct{}) - flushed := make(chan struct{}) - go func() { - defer close(flushed) - for { - select { - case <-stop: - return - default: - if err := db.flushHits(); err != nil { - t.Errorf("flushHits failed: %v", err) - return - } - } - } - }() - var wg sync.WaitGroup - for range writers { - wg.Go(func() { - for range hitsEach { - if err := db.RecordArtifactHit(versionPURL, filename); err != nil { - t.Errorf("RecordArtifactHit failed: %v", err) - } - } - }) - } - wg.Wait() - close(stop) - <-flushed - - if err := db.flushHits(); err != nil { - t.Fatalf("final flushHits failed: %v", err) - } - if got := hitCount(t, db, versionPURL, filename); got != writers*hitsEach { - t.Errorf("hit count = %d, want %d", got, writers*hitsEach) - } - }) -} - -func TestBatchHitsLogsHitsDroppedOnClose(t *testing.T) { - db := createTestDB(t) - versionPURL, filename := seedHitTestArtifact(t, db) - var logs bytes.Buffer - db.BatchHits(time.Hour, slog.New(slog.NewTextHandler(&logs, nil))) - - _ = db.RecordArtifactHit(versionPURL, filename) - _ = db.DB.Close() - _ = db.Close() - - if !strings.Contains(logs.String(), "dropping them") { - t.Errorf("no error logged for hits dropped on close, logs: %q", logs.String()) - } -} - -func TestCloseTwiceWithBatchHits(t *testing.T) { - db := createTestDB(t) - db.BatchHits(time.Hour, discardLogger) - _ = db.Close() - _ = db.Close() -} diff --git a/internal/database/local_artifacts.go b/internal/database/local_artifacts.go deleted file mode 100644 index 162b471..0000000 --- a/internal/database/local_artifacts.go +++ /dev/null @@ -1,126 +0,0 @@ -package database - -import ( - "database/sql" - "fmt" - "time" -) - -// LocalArtifact is a pushed artifact with the integrity of its version. -type LocalArtifact struct { - Artifact - Integrity sql.NullString `db:"integrity"` -} - -// GetLocalArtifacts returns the artifacts pushed to the proxy for a package, -// ordered by version and filename. -func (db *DB) GetLocalArtifacts(packagePURL string) ([]LocalArtifact, error) { - var artifacts []LocalArtifact - query := db.Rebind(` - SELECT a.id, a.version_purl, a.filename, a.upstream_url, a.storage_path, a.content_hash, - a.size, a.content_type, a.fetched_at, a.hit_count, a.last_accessed_at, - a.created_at, a.updated_at, a.source, a.manifest, v.integrity - FROM artifacts a - JOIN versions v ON v.purl = a.version_purl - WHERE v.package_purl = ? AND a.source = 'local' - ORDER BY a.version_purl, a.filename - `) - if err := db.Select(&artifacts, query, packagePURL); err != nil { - return nil, err - } - return artifacts, nil -} - -// DeleteLocalArtifacts removes the pushed artifacts of a version, or only the -// one named filename when it is set, and queues their stored objects for -// deletion in the same transaction, so a failed queue keeps the records -// instead of orphaning the objects. Artifacts cached from an upstream are left -// alone. It returns how many records it removed. -func (db *DB) DeleteLocalArtifacts(versionPURL, filename string) (int, error) { - tx, err := db.Beginx() - if err != nil { - return 0, err - } - defer func() { _ = tx.Rollback() }() - - where := "version_purl = ? AND source = 'local'" - args := []any{versionPURL} - if filename != "" { - where += " AND filename = ?" - args = append(args, filename) - } - - var paths []string - selectQuery := db.Rebind(`SELECT storage_path FROM artifacts WHERE ` + where + ` AND storage_path IS NOT NULL`) - if err := tx.Select(&paths, selectQuery, args...); err != nil { - return 0, fmt.Errorf("reading local artifacts: %w", err) - } - - res, err := tx.Exec(db.Rebind(`DELETE FROM artifacts WHERE `+where), args...) - if err != nil { - return 0, fmt.Errorf("deleting local artifacts: %w", err) - } - deleted, err := res.RowsAffected() - if err != nil { - return 0, err - } - - queuedAt := time.Now().UTC() - for _, path := range paths { - if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), path, queuedAt); err != nil { - return 0, fmt.Errorf("queueing %s for deletion: %w", path, err) - } - } - return int(deleted), tx.Commit() -} - -// RecordLocalArtifact records a pushed artifact in one transaction: the -// package if it is missing, the version, and the artifact. An artifact cached -// from an upstream under the same name is replaced and its stored object -// queued for deletion; a pushed one is never replaced, and the whole record -// is refused with ErrLocalArtifactExists, so of two concurrent pushes of the -// same file exactly one wins. -// -// When oneFilePerVersion is false, an existing version record is left as it -// is. Otherwise its integrity is set to v.Integrity, and the push is refused -// if the version already holds a pushed artifact under any filename, since -// aliases of one name (Composer's are case-insensitive) share the version -// record and its integrity. The version upsert locks that record first, so -// concurrent pushes of aliases see each other. -func (db *DB) RecordLocalArtifact(pkg *Package, v *Version, oneFilePerVersion bool, a *Artifact) error { - local := *a - local.Source = SourceLocal - return db.recordArtifact(&local, func(tx execer) error { - now := time.Now() - if _, err := tx.Exec(db.Rebind(` - INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(purl) DO NOTHING - `), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now); err != nil { - return fmt.Errorf("inserting package: %w", err) - } - versionConflict := `DO NOTHING` - if oneFilePerVersion { - versionConflict = `DO UPDATE SET integrity = excluded.integrity, updated_at = excluded.updated_at` - } - if _, err := tx.Exec(db.Rebind(` - INSERT INTO versions (purl, package_purl, integrity, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(purl) `+versionConflict, - ), v.PURL, v.PackagePURL, v.Integrity, now, now); err != nil { - return fmt.Errorf("recording version: %w", err) - } - if oneFilePerVersion { - var pushed int - if err := tx.Get(&pushed, db.Rebind(` - SELECT COUNT(*) FROM artifacts WHERE version_purl = ? AND source = ? - `), v.PURL, SourceLocal); err != nil { - return fmt.Errorf("checking pushed artifacts: %w", err) - } - if pushed > 0 { - return ErrLocalArtifactExists - } - } - return nil - }, ErrLocalArtifactExists) -} diff --git a/internal/database/local_artifacts_test.go b/internal/database/local_artifacts_test.go deleted file mode 100644 index a6d0ed5..0000000 --- a/internal/database/local_artifacts_test.go +++ /dev/null @@ -1,507 +0,0 @@ -package database - -import ( - "database/sql" - "errors" - "fmt" - "path/filepath" - "sync" - "testing" - "time" -) - -func seedSourcedArtifact(t *testing.T, db *DB, name, source string, size int64, fetched time.Time) { - t.Helper() - pkgPURL := "pkg:npm/" + name - versionPURL := pkgPURL + "@1.0.0" - if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: "npm", Name: name}); err != nil { - t.Fatalf("UpsertPackage: %v", err) - } - if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - art := &Artifact{ - VersionPURL: versionPURL, - Filename: name + "-1.0.0.tgz", - StoragePath: sql.NullString{String: "npm/" + name + "/1.0.0/" + name + "-1.0.0.tgz", Valid: true}, - Size: sql.NullInt64{Int64: size, Valid: true}, - FetchedAt: sql.NullTime{Time: fetched, Valid: true}, - Source: source, - } - if source != SourceLocal { - art.UpstreamURL = "https://registry.example/" + name + "-1.0.0.tgz" - } - if err := db.UpsertArtifact(art); err != nil { - t.Fatalf("UpsertArtifact: %v", err) - } -} - -func TestArtifactSourceDefaultsToUpstream(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - seedSourcedArtifact(t, db, "cached", "", 100, time.Now()) - seedSourcedArtifact(t, db, "pushed", SourceLocal, 100, time.Now()) - - cached, err := db.GetArtifact("pkg:npm/cached@1.0.0", "cached-1.0.0.tgz") - if err != nil { - t.Fatalf("GetArtifact(cached): %v", err) - } - if cached.Source != SourceUpstream { - t.Errorf("cached artifact source = %q, want %q", cached.Source, SourceUpstream) - } - - pushed, err := db.GetArtifact("pkg:npm/pushed@1.0.0", "pushed-1.0.0.tgz") - if err != nil { - t.Fatalf("GetArtifact(pushed): %v", err) - } - if pushed.Source != SourceLocal { - t.Errorf("pushed artifact source = %q, want %q", pushed.Source, SourceLocal) - } - if pushed.UpstreamURL != "" { - t.Errorf("pushed artifact upstream_url = %q, want empty", pushed.UpstreamURL) - } - }) -} - -func TestLocalArtifactsAreNotEvictable(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - seedSourcedArtifact(t, db, "cached", SourceUpstream, 1000, time.Now()) - seedSourcedArtifact(t, db, "pushed", SourceLocal, 5000, time.Now()) - - total, err := db.GetTotalCacheSize() - if err != nil { - t.Fatalf("GetTotalCacheSize: %v", err) - } - if total != 1000 { - t.Errorf("total cache size = %d, want 1000 (local artifacts excluded)", total) - } - - lru, err := db.GetLeastRecentlyUsedArtifacts(10) - if err != nil { - t.Fatalf("GetLeastRecentlyUsedArtifacts: %v", err) - } - if len(lru) != 1 || lru[0].Filename != "cached-1.0.0.tgz" { - t.Errorf("LRU artifacts = %+v, want only cached-1.0.0.tgz", lru) - } - }) -} - -func TestLocalArtifactsAreNotRetentionCandidates(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - old := time.Now().Add(-48 * time.Hour) - cutoff := time.Now().Add(-24 * time.Hour) - seedSourcedArtifact(t, db, "cached", SourceUpstream, 100, old) - seedSourcedArtifact(t, db, "pushed", SourceLocal, 100, old) - - maxID, err := db.MaxArtifactID() - if err != nil { - t.Fatalf("MaxArtifactID: %v", err) - } - candidates, err := db.GetRetentionCandidates(0, maxID, cutoff) - if err != nil { - t.Fatalf("GetRetentionCandidates: %v", err) - } - got := candidateNames(candidates) - if _, ok := got["pushed-1.0.0.tgz"]; ok { - t.Error("local artifact returned as a retention candidate") - } - if _, ok := got["cached-1.0.0.tgz"]; !ok { - t.Error("upstream artifact missing from retention candidates") - } - - cleared, err := db.ClearExpiredArtifact("pkg:npm/pushed@1.0.0", "pushed-1.0.0.tgz", "npm/pushed/1.0.0/pushed-1.0.0.tgz", cutoff) - if err != nil || cleared { - t.Errorf("ClearExpiredArtifact(local) = %v, %v, want false, nil", cleared, err) - } - }) -} - -func TestMigrationAddsArtifactSource(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "existing.db") - sqlDB, err := sql.Open("sqlite", dbPath) - if err != nil { - t.Fatalf("open: %v", err) - } - if _, err := sqlDB.Exec(schemaSQLite); err != nil { - t.Fatalf("create schema: %v", err) - } - if _, err := sqlDB.Exec(`ALTER TABLE artifacts DROP COLUMN source`); err != nil { - t.Fatalf("drop source column: %v", err) - } - if _, err := sqlDB.Exec(`DELETE FROM migrations WHERE name = '012_add_artifact_source'`); err != nil { - t.Fatalf("forget migration: %v", err) - } - if _, err := sqlDB.Exec(`INSERT INTO artifacts (version_purl, filename, upstream_url, created_at, updated_at) - VALUES ('pkg:npm/old@1.0.0', 'old-1.0.0.tgz', 'https://registry.example/old-1.0.0.tgz', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err != nil { - t.Fatalf("seed artifact: %v", err) - } - if err := sqlDB.Close(); err != nil { - t.Fatalf("close: %v", err) - } - - db, err := Open(dbPath) - if err != nil { - t.Fatalf("Open: %v", err) - } - defer func() { _ = db.Close() }() - if err := db.MigrateSchema(); err != nil { - t.Fatalf("MigrateSchema: %v", err) - } - - art, err := db.GetArtifact("pkg:npm/old@1.0.0", "old-1.0.0.tgz") - if err != nil { - t.Fatalf("GetArtifact: %v", err) - } - if art.Source != SourceUpstream { - t.Errorf("existing artifact source = %q, want %q", art.Source, SourceUpstream) - } -} - -func TestGetLocalArtifacts(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - seedSourcedArtifact(t, db, "request", SourceLocal, 100, time.Now()) - seedSourcedArtifact(t, db, "lodash", SourceLocal, 100, time.Now()) - seedSourcedArtifact(t, db, "express", SourceUpstream, 100, time.Now()) - - local, err := db.GetLocalArtifacts("pkg:npm/request") - if err != nil { - t.Fatalf("GetLocalArtifacts: %v", err) - } - if len(local) != 1 || local[0].Filename != "request-1.0.0.tgz" { - t.Fatalf("local artifacts of request = %+v, want only request-1.0.0.tgz", local) - } - - upstreamOnly, err := db.GetLocalArtifacts("pkg:npm/express") - if err != nil { - t.Fatalf("GetLocalArtifacts(express): %v", err) - } - if len(upstreamOnly) != 0 { - t.Errorf("local artifacts of express = %+v, want none", upstreamOnly) - } - }) -} - -func TestDeleteLocalArtifacts(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - seedSourcedArtifact(t, db, "request", SourceLocal, 100, time.Now()) - seedSourcedArtifact(t, db, "express", SourceUpstream, 100, time.Now()) - const pushedPath = "npm/request/1.0.0/request-1.0.0.tgz" - - deleted, err := db.DeleteLocalArtifacts("pkg:npm/request@1.0.0", "") - if err != nil || deleted != 1 { - t.Fatalf("DeleteLocalArtifacts(request) = %d, %v, want 1, nil", deleted, err) - } - art, err := db.GetArtifact("pkg:npm/request@1.0.0", "request-1.0.0.tgz") - if err != nil || art != nil { - t.Errorf("GetArtifact after delete = %+v, %v, want nil, nil", art, err) - } - var queued int - if err := db.Get(&queued, db.Rebind(`SELECT COUNT(*) FROM pending_deletes WHERE path = ?`), pushedPath); err != nil || queued != 1 { - t.Errorf("queued deletes for %s = %d, %v, want 1", pushedPath, queued, err) - } - - deleted, err = db.DeleteLocalArtifacts("pkg:npm/express@1.0.0", "") - if err != nil || deleted != 0 { - t.Errorf("DeleteLocalArtifacts(upstream) = %d, %v, want 0, nil", deleted, err) - } - if art, err := db.GetArtifact("pkg:npm/express@1.0.0", "express-1.0.0.tgz"); err != nil || art == nil { - t.Errorf("upstream artifact removed by DeleteLocalArtifacts: %+v, %v", art, err) - } - }) -} - -func TestLocalArtifactManifestAndIntegrity(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - if err := db.UpsertPackage(&Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"}); err != nil { - t.Fatalf("UpsertPackage: %v", err) - } - if err := db.UpsertVersion(&Version{ - PURL: "pkg:npm/request@2.88.3", - PackagePURL: "pkg:npm/request", - Integrity: sql.NullString{String: "sha512-abc", Valid: true}, - }); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - const manifest = `{"name":"request","version":"2.88.3"}` - if err := db.UpsertArtifact(&Artifact{ - VersionPURL: "pkg:npm/request@2.88.3", - Filename: "request-2.88.3.tgz", - StoragePath: sql.NullString{String: "npm/request/2.88.3/x/request-2.88.3.tgz", Valid: true}, - ContentHash: sql.NullString{String: "00", Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - Source: SourceLocal, - Manifest: sql.NullString{String: manifest, Valid: true}, - }); err != nil { - t.Fatalf("UpsertArtifact: %v", err) - } - - local, err := db.GetLocalArtifacts("pkg:npm/request") - if err != nil || len(local) != 1 { - t.Fatalf("GetLocalArtifacts = %+v, %v", local, err) - } - if local[0].Manifest.String != manifest || local[0].Integrity.String != "sha512-abc" { - t.Errorf("manifest/integrity = %q/%q", local[0].Manifest.String, local[0].Integrity.String) - } - }) -} - -func TestMigrationAddsArtifactManifest(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "existing.db") - sqlDB, err := sql.Open("sqlite", dbPath) - if err != nil { - t.Fatalf("open: %v", err) - } - if _, err := sqlDB.Exec(schemaSQLite); err != nil { - t.Fatalf("create schema: %v", err) - } - if _, err := sqlDB.Exec(`ALTER TABLE artifacts DROP COLUMN manifest`); err != nil { - t.Fatalf("drop manifest column: %v", err) - } - if _, err := sqlDB.Exec(`DELETE FROM migrations WHERE name = '013_add_artifact_manifest'`); err != nil { - t.Fatalf("forget migration: %v", err) - } - if err := sqlDB.Close(); err != nil { - t.Fatalf("close: %v", err) - } - - db, err := Open(dbPath) - if err != nil { - t.Fatalf("Open: %v", err) - } - defer func() { _ = db.Close() }() - if err := db.MigrateSchema(); err != nil { - t.Fatalf("MigrateSchema: %v", err) - } - has, err := db.HasColumn("artifacts", "manifest") - if err != nil || !has { - t.Errorf("artifacts.manifest after migration = %v, %v", has, err) - } -} - -func TestUpsertPackageKeepsSource(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - pkg := &Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"} - if err := db.UpsertPackage(pkg); err != nil { - t.Fatalf("UpsertPackage: %v", err) - } - if _, err := db.Exec(db.Rebind(`UPDATE packages SET source = ? WHERE purl = ?`), "deps.dev", pkg.PURL); err != nil { - t.Fatalf("setting source: %v", err) - } - if err := db.UpsertPackage(pkg); err != nil { - t.Fatalf("second UpsertPackage: %v", err) - } - got, err := db.GetPackageByPURL(pkg.PURL) - if err != nil || got == nil { - t.Fatalf("GetPackageByPURL: %+v, %v", got, err) - } - if got.Source.String != "deps.dev" { - t.Errorf("source after upsert = %q, want deps.dev", got.Source.String) - } - }) -} - -func localPush(versionPURL, filename, storagePath string) *Artifact { - return &Artifact{ - VersionPURL: versionPURL, - Filename: filename, - StoragePath: sql.NullString{String: storagePath, Valid: true}, - ContentHash: sql.NullString{String: storagePath, Valid: true}, - Size: sql.NullInt64{Int64: 1, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - } -} - -func TestRecordLocalArtifactNeverReplacesAPush(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - pkg := &Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"} - version := func(integrity string) *Version { - return &Version{PURL: "pkg:npm/request@1.0.0", PackagePURL: pkg.PURL, - Integrity: sql.NullString{String: integrity, Valid: true}} - } - - first := localPush("pkg:npm/request@1.0.0", "request-1.0.0.tgz", "npm/request/first") - if err := db.RecordLocalArtifact(pkg, version("sha512-first"), true, first); err != nil { - t.Fatalf("first RecordLocalArtifact: %v", err) - } - second := localPush("pkg:npm/request@1.0.0", "request-1.0.0.tgz", "npm/request/second") - if err := db.RecordLocalArtifact(pkg, version("sha512-second"), true, second); !errors.Is(err, ErrLocalArtifactExists) { - t.Fatalf("second RecordLocalArtifact error = %v, want ErrLocalArtifactExists", err) - } - - art, err := db.GetArtifact("pkg:npm/request@1.0.0", "request-1.0.0.tgz") - if err != nil || art == nil || art.StoragePath.String != "npm/request/first" || art.Source != SourceLocal { - t.Fatalf("artifact after refused push = %+v, %v, want the first push", art, err) - } - v, err := db.GetVersionByPURL("pkg:npm/request@1.0.0") - if err != nil || v == nil || v.Integrity.String != "sha512-first" { - t.Errorf("version integrity after refused push = %+v, %v, want the first push's", v, err) - } - var queued int - if err := db.Get(&queued, db.Rebind(`SELECT COUNT(*) FROM pending_deletes`)); err != nil || queued != 0 { - t.Errorf("queued deletes = %d, %v, want none", queued, err) - } - }) -} - -func TestRecordLocalArtifactReplacesUpstreamCopy(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - seedSourcedArtifact(t, db, "request", SourceUpstream, 100, time.Now()) - pkg := &Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"} - v := &Version{PURL: "pkg:npm/request@1.0.0", PackagePURL: pkg.PURL} - - pushed := localPush("pkg:npm/request@1.0.0", "request-1.0.0.tgz", "npm/request/pushed") - if err := db.RecordLocalArtifact(pkg, v, true, pushed); err != nil { - t.Fatalf("RecordLocalArtifact: %v", err) - } - art, err := db.GetArtifact("pkg:npm/request@1.0.0", "request-1.0.0.tgz") - if err != nil || art == nil || art.Source != SourceLocal || art.StoragePath.String != "npm/request/pushed" { - t.Fatalf("artifact = %+v, %v, want the push", art, err) - } - const upstreamPath = "npm/request/1.0.0/request-1.0.0.tgz" - var queued int - if err := db.Get(&queued, db.Rebind(`SELECT COUNT(*) FROM pending_deletes WHERE path = ?`), upstreamPath); err != nil || queued != 1 { - t.Errorf("queued deletes for the upstream copy = %d, %v, want 1", queued, err) - } - }) -} - -func TestRecordLocalArtifactCanKeepTheVersionRecord(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - pkg := &Package{PURL: "pkg:generic/tools", Ecosystem: "generic", Name: "tools"} - if err := db.UpsertPackage(pkg); err != nil { - t.Fatalf("UpsertPackage: %v", err) - } - if err := db.UpsertVersion(&Version{PURL: "pkg:generic/tools@1.2.0", PackagePURL: pkg.PURL, - License: sql.NullString{String: "MIT", Valid: true}, - Integrity: sql.NullString{String: "sha256-upstream", Valid: true}}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - - pushed := localPush("pkg:generic/tools@1.2.0", "tool.tar.gz", "generic/tools/pushed") - if err := db.RecordLocalArtifact(pkg, &Version{PURL: "pkg:generic/tools@1.2.0", PackagePURL: pkg.PURL}, false, pushed); err != nil { - t.Fatalf("RecordLocalArtifact: %v", err) - } - v, err := db.GetVersionByPURL("pkg:generic/tools@1.2.0") - if err != nil || v == nil || v.Integrity.String != "sha256-upstream" || v.License.String != "MIT" { - t.Errorf("version after push = %+v, %v, want it unchanged", v, err) - } - }) -} - -func TestRecordLocalArtifactConcurrentPushesHaveOneWinner(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - pkg := &Package{PURL: "pkg:generic/tools", Ecosystem: "generic", Name: "tools"} - const pushes = 8 - start := make(chan struct{}) - errs := make([]error, pushes) - var wg sync.WaitGroup - for i := range pushes { - wg.Go(func() { - <-start - v := &Version{PURL: "pkg:generic/tools@1.2.0", PackagePURL: pkg.PURL} - a := localPush("pkg:generic/tools@1.2.0", "tool.tar.gz", fmt.Sprintf("generic/tools/push-%d", i)) - errs[i] = db.RecordLocalArtifact(pkg, v, false, a) - }) - } - close(start) - wg.Wait() - - winners := 0 - for i, err := range errs { - switch { - case err == nil: - winners++ - case !errors.Is(err, ErrLocalArtifactExists): - t.Errorf("push %d error = %v, want ErrLocalArtifactExists", i, err) - } - } - if winners != 1 { - t.Errorf("%d pushes succeeded, want exactly 1", winners) - } - }) -} - -// Aliases of one name, such as Composer names differing in case, share the -// version record and its integrity, so only one of them may be pushed. -func TestRecordLocalArtifactOneFilePerVersionRefusesAliases(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - pkg := &Package{PURL: "pkg:composer/acme/private", Ecosystem: "composer", Name: "acme/private"} - const pushes = 8 - start := make(chan struct{}) - errs := make([]error, pushes) - var wg sync.WaitGroup - for i := range pushes { - wg.Go(func() { - <-start - v := &Version{PURL: "pkg:composer/acme/private@1.0.0", PackagePURL: pkg.PURL, - Integrity: sql.NullString{String: fmt.Sprintf("sha512-push-%d", i), Valid: true}} - a := localPush("pkg:composer/acme/private@1.0.0", fmt.Sprintf("alias-%d-1.0.0.zip", i), - fmt.Sprintf("composer/acme/private/push-%d", i)) - errs[i] = db.RecordLocalArtifact(pkg, v, true, a) - }) - } - close(start) - wg.Wait() - - winner := -1 - for i, err := range errs { - switch { - case err == nil: - if winner >= 0 { - t.Errorf("pushes %d and %d both succeeded, want exactly 1", winner, i) - } - winner = i - case !errors.Is(err, ErrLocalArtifactExists): - t.Errorf("push %d error = %v, want ErrLocalArtifactExists", i, err) - } - } - if winner < 0 { - t.Fatal("no push succeeded") - } - artifacts, err := db.GetArtifactsByVersionPURL("pkg:composer/acme/private@1.0.0") - if err != nil { - t.Fatal(err) - } - if len(artifacts) != 1 || artifacts[0].Filename != fmt.Sprintf("alias-%d-1.0.0.zip", winner) { - t.Errorf("artifacts = %+v, want only push %d", artifacts, winner) - } - v, err := db.GetVersionByPURL("pkg:composer/acme/private@1.0.0") - if err != nil || v == nil || v.Integrity.String != fmt.Sprintf("sha512-push-%d", winner) { - t.Errorf("version = %+v, %v, want the integrity of push %d", v, err, winner) - } - }) -} - -func TestRecordCachedArtifactNeverReplacesAPush(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - pkg := &Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"} - pushed := localPush("pkg:npm/request@1.0.0", "request-1.0.0.tgz", "npm/request/pushed") - if err := db.RecordLocalArtifact(pkg, &Version{PURL: "pkg:npm/request@1.0.0", PackagePURL: pkg.PURL, - Integrity: sql.NullString{String: "sha512-pushed", Valid: true}}, true, pushed); err != nil { - t.Fatalf("RecordLocalArtifact: %v", err) - } - - fetched := &Artifact{ - VersionPURL: "pkg:npm/request@1.0.0", - Filename: "request-1.0.0.tgz", - UpstreamURL: "https://registry.example/request-1.0.0.tgz", - StoragePath: sql.NullString{String: "npm/request/fetched", Valid: true}, - } - err := db.RecordCachedArtifact(pkg, &Version{PURL: "pkg:npm/request@1.0.0", PackagePURL: pkg.PURL, - Integrity: sql.NullString{String: "sha512-upstream", Valid: true}}, fetched) - if !errors.Is(err, ErrLocalArtifactExists) { - t.Fatalf("RecordCachedArtifact error = %v, want ErrLocalArtifactExists", err) - } - if err := db.UpsertArtifact(fetched); !errors.Is(err, ErrLocalArtifactExists) { - t.Errorf("UpsertArtifact error = %v, want ErrLocalArtifactExists", err) - } - - art, err := db.GetArtifact("pkg:npm/request@1.0.0", "request-1.0.0.tgz") - if err != nil || art == nil || art.Source != SourceLocal || art.StoragePath.String != "npm/request/pushed" { - t.Errorf("artifact = %+v, %v, want the push", art, err) - } - v, err := db.GetVersionByPURL("pkg:npm/request@1.0.0") - if err != nil || v == nil || v.Integrity.String != "sha512-pushed" { - t.Errorf("version = %+v, %v, want the push's integrity", v, err) - } - }) -} diff --git a/internal/database/lock.go b/internal/database/lock.go deleted file mode 100644 index f4b1f70..0000000 --- a/internal/database/lock.go +++ /dev/null @@ -1,107 +0,0 @@ -package database - -import ( - "context" - "database/sql" - "database/sql/driver" - "fmt" - "hash/fnv" - "time" -) - -// Several instances can share one Postgres database. Advisory locks keep -// them from creating the schema, migrating it or running a background job at -// the same time. SQLite is single-instance, so there every lock is granted -// at once. - -// lockKey maps a lock name to the 64-bit key Postgres advisory locks take. -func lockKey(name string) int64 { - h := fnv.New64a() - _, _ = h.Write([]byte("git-pkgs/proxy:" + name)) - return int64(h.Sum64()) //nolint:gosec // a key, wrapping is fine -} - -// WithLock runs fn while holding the named lock, waiting for another -// instance that holds it. -func (db *DB) WithLock(ctx context.Context, name string, fn func() error) error { - if db.dialect != DialectPostgres { - return fn() - } - conn, err := db.Conn(ctx) - if err != nil { - return fmt.Errorf("taking connection for lock %s: %w", name, err) - } - defer func() { _ = conn.Close() }() - if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_lock($1)", lockKey(name)); err != nil { - return fmt.Errorf("taking lock %s: %w", name, err) - } - defer unlock(conn, name) - return fn() -} - -// TryLock takes the named lock if no other instance holds it. When ok is -// true the caller must call release once done. The lock lives in a -// connection held for that time, and Postgres frees it if the process dies. -func (db *DB) TryLock(ctx context.Context, name string) (release func(), ok bool, err error) { - if db.dialect != DialectPostgres { - return func() {}, true, nil - } - conn, err := db.Conn(ctx) - if err != nil { - return nil, false, fmt.Errorf("taking connection for lock %s: %w", name, err) - } - if err := conn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock($1)", lockKey(name)).Scan(&ok); err != nil { - _ = conn.Close() - return nil, false, fmt.Errorf("taking lock %s: %w", name, err) - } - if !ok { - _ = conn.Close() - return nil, false, nil - } - return func() { - unlock(conn, name) - _ = conn.Close() - }, true, nil -} - -// unlockTimeout bounds the unlock. If it fails, closing the connection still -// ends the session that holds the lock. -const unlockTimeout = 10 * time.Second - -func unlock(conn *sql.Conn, name string) { - ctx, cancel := context.WithTimeout(context.Background(), unlockTimeout) - defer cancel() - if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", lockKey(name)); err != nil { - // Drop the connection so the session, and with it the lock, ends. - _ = conn.Raw(func(any) error { return driver.ErrBadConn }) - } -} - -// ClaimJobRun records that the named job starts now unless it started less -// than minGap ago, and reports whether the caller should run it. Instances -// sharing the database each keep their own timer for a scheduled job, and -// this lets only the first of them run it per interval. -func (db *DB) ClaimJobRun(name string, minGap time.Duration) (bool, error) { - now := time.Now().UTC() - res, err := db.Exec(db.Rebind(` - INSERT INTO job_runs (name, started_at) VALUES (?, ?) - ON CONFLICT(name) DO UPDATE SET started_at = excluded.started_at - WHERE job_runs.started_at <= ?`), name, now, now.Add(-minGap)) - if err != nil { - return false, fmt.Errorf("claiming job run %s: %w", name, err) - } - n, err := res.RowsAffected() - if err != nil { - return false, fmt.Errorf("claiming job run %s: %w", name, err) - } - return n > 0, nil -} - -// ReleaseJobRun drops the named job's last claim, so the next attempt on any -// instance runs it. It is for a run that was interrupted before it finished. -func (db *DB) ReleaseJobRun(name string) error { - if _, err := db.Exec(db.Rebind(`DELETE FROM job_runs WHERE name = ?`), name); err != nil { - return fmt.Errorf("releasing job run %s: %w", name, err) - } - return nil -} diff --git a/internal/database/lock_test.go b/internal/database/lock_test.go deleted file mode 100644 index de3e64c..0000000 --- a/internal/database/lock_test.go +++ /dev/null @@ -1,347 +0,0 @@ -package database - -import ( - "context" - "net/url" - "os" - "strings" - "sync" - "sync/atomic" - "testing" - "time" -) - -func TestTryLockSQLiteAlwaysGranted(t *testing.T) { - db := createTestDB(t) - defer func() { _ = db.Close() }() - release, ok, err := db.TryLock(context.Background(), "job:test") - if err != nil || !ok { - t.Fatalf("TryLock = %v, %v; want granted", ok, err) - } - // SQLite is single-instance, so a second holder is granted too. - release2, ok, err := db.TryLock(context.Background(), "job:test") - if err != nil || !ok { - t.Fatalf("second TryLock = %v, %v; want granted", ok, err) - } - release2() - release() -} - -func TestClaimJobRun(t *testing.T) { - forEachDialect(t, func(t *testing.T, db *DB) { - claimed, err := db.ClaimJobRun("mirror-schedule:a", time.Hour) - if err != nil || !claimed { - t.Fatalf("first claim = %v, %v; want claimed", claimed, err) - } - claimed, err = db.ClaimJobRun("mirror-schedule:a", time.Hour) - if err != nil || claimed { - t.Fatalf("claim within the gap = %v, %v; want not claimed", claimed, err) - } - claimed, err = db.ClaimJobRun("mirror-schedule:b", time.Hour) - if err != nil || !claimed { - t.Fatalf("claim of another job = %v, %v; want claimed", claimed, err) - } - // A run started longer ago than the gap can be claimed again. - claimed, err = db.ClaimJobRun("mirror-schedule:a", 0) - if err != nil || !claimed { - t.Fatalf("claim after the gap = %v, %v; want claimed", claimed, err) - } - // A released run can be claimed again within the gap. - if err := db.ReleaseJobRun("mirror-schedule:a"); err != nil { - t.Fatal(err) - } - claimed, err = db.ClaimJobRun("mirror-schedule:a", time.Hour) - if err != nil || !claimed { - t.Fatalf("claim after release = %v, %v; want claimed", claimed, err) - } - claimed, err = db.ClaimJobRun("mirror-schedule:b", time.Hour) - if err != nil || claimed { - t.Fatalf("other job after release = %v, %v; want still claimed", claimed, err) - } - }) -} - -// forEachDialect runs fn against SQLite and, when PROXY_DATABASE_URL is set, -// against Postgres. -func forEachDialect(t *testing.T, fn func(t *testing.T, db *DB)) { - t.Run("sqlite", func(t *testing.T) { - db := createTestDB(t) - defer func() { _ = db.Close() }() - fn(t, db) - }) - t.Run("postgres", func(t *testing.T) { - db := createTestPostgresDB(t) - if db == nil { - t.Skip("PROXY_DATABASE_URL not set") - } - defer func() { _ = db.Close() }() - fn(t, db) - }) -} - -func openTestPostgres(t *testing.T) *DB { - t.Helper() - dbURL := os.Getenv("PROXY_DATABASE_URL") - if dbURL == "" { - t.Skip("PROXY_DATABASE_URL not set") - } - db, err := OpenPostgres(dbURL) - if err != nil { - t.Fatalf("OpenPostgres: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - return db -} - -func TestTryLockPostgresExcludesOtherInstance(t *testing.T) { - a := openTestPostgres(t) - b := openTestPostgres(t) - ctx := context.Background() - - release, ok, err := a.TryLock(ctx, "job:test-exclusive") - if err != nil || !ok { - t.Fatalf("TryLock on a = %v, %v; want granted", ok, err) - } - if _, ok, err := b.TryLock(ctx, "job:test-exclusive"); err != nil || ok { - t.Fatalf("TryLock on b while a holds it = %v, %v; want refused", ok, err) - } - // Another lock name is independent. - releaseOther, ok, err := b.TryLock(ctx, "job:test-other") - if err != nil || !ok { - t.Fatalf("TryLock on another name = %v, %v; want granted", ok, err) - } - releaseOther() - - release() - release, ok, err = b.TryLock(ctx, "job:test-exclusive") - if err != nil || !ok { - t.Fatalf("TryLock on b after a released = %v, %v; want granted", ok, err) - } - release() -} - -func TestTryLockPostgresFreedWhenSessionEnds(t *testing.T) { - a := openTestPostgres(t) - b := openTestPostgres(t) - ctx := context.Background() - - if _, ok, err := a.TryLock(ctx, "job:test-crash"); err != nil || !ok { - t.Fatalf("TryLock on a = %v, %v; want granted", ok, err) - } - // End a's session holding the lock, as a crash would. - if _, err := b.Exec(`SELECT pg_terminate_backend(pid) FROM pg_locks - WHERE locktype = 'advisory' AND pid <> pg_backend_pid()`); err != nil { - t.Fatalf("terminating a's session: %v", err) - } - waitUntil(t, func() bool { - release, ok, err := b.TryLock(ctx, "job:test-crash") - if ok { - release() - } - return err == nil && ok - }) -} - -func TestWithLockPostgresSerialises(t *testing.T) { - const instances = 4 - dbs := make([]*DB, instances) - for i := range dbs { - dbs[i] = openTestPostgres(t) - } - var inside, overlapped atomic.Int32 - var wg sync.WaitGroup - for _, db := range dbs { - wg.Go(func() { - err := db.WithLock(context.Background(), "test-serialise", func() error { - if inside.Add(1) > 1 { - overlapped.Store(1) - } - time.Sleep(20 * time.Millisecond) - inside.Add(-1) - return nil - }) - if err != nil { - t.Errorf("WithLock: %v", err) - } - }) - } - wg.Wait() - if overlapped.Load() != 0 { - t.Error("two instances held the lock at once") - } -} - -// TestOpenPostgresOrCreateConcurrent starts several instances at once on an -// empty database. Without the schema lock all but one failed on each -// other's tables. -func TestOpenPostgresOrCreateConcurrent(t *testing.T) { - admin := openTestPostgres(t) - name := "proxy_lock_test_" + strings.ReplaceAll(time.Now().UTC().Format("150405.000000"), ".", "_") - if _, err := admin.Exec("CREATE DATABASE " + name); err != nil { - t.Skipf("cannot create a scratch database: %v", err) - } - t.Cleanup(func() { _, _ = admin.Exec("DROP DATABASE IF EXISTS " + name + " WITH (FORCE)") }) - - u, err := url.Parse(os.Getenv("PROXY_DATABASE_URL")) - if err != nil { - t.Fatalf("parsing PROXY_DATABASE_URL: %v", err) - } - u.Path = "/" + name - scratch := u.String() - - const instances = 5 - errs := make([]error, instances) - var wg sync.WaitGroup - start := make(chan struct{}) - for i := range instances { - wg.Go(func() { - <-start - db, err := OpenPostgresOrCreate(scratch) - if err == nil { - err = db.MigrateSchema() - _ = db.Close() - } - errs[i] = err - }) - } - close(start) - wg.Wait() - for i, err := range errs { - if err != nil { - t.Errorf("instance %d: %v", i, err) - } - } - - db, err := OpenPostgres(scratch) - if err != nil { - t.Fatalf("OpenPostgres: %v", err) - } - defer func() { _ = db.Close() }() - var n int - if err := db.Get(&n, "SELECT COUNT(*) FROM schema_info"); err != nil || n != 1 { - t.Errorf("schema_info rows = %d, %v; want 1", n, err) - } - if err := db.Get(&n, "SELECT COUNT(*) FROM migrations"); err != nil || n != len(migrations) { - t.Errorf("migrations rows = %d, %v; want %d", n, err, len(migrations)) - } -} - -func waitUntil(t *testing.T, cond func() bool) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for !cond() { - if time.Now().After(deadline) { - t.Fatal("timed out") - } - time.Sleep(10 * time.Millisecond) - } -} - -func TestMigrateAddsJobRuns(t *testing.T) { - forEachDialect(t, func(t *testing.T, db *DB) { - // A database from before the job_runs migration. - if _, err := db.Exec("DROP TABLE job_runs"); err != nil { - t.Fatalf("dropping job_runs: %v", err) - } - if _, err := db.Exec(db.Rebind("DELETE FROM migrations WHERE name = ?"), "015_add_job_runs"); err != nil { - t.Fatalf("forgetting the migration: %v", err) - } - if err := db.MigrateSchema(); err != nil { - t.Fatalf("MigrateSchema: %v", err) - } - if claimed, err := db.ClaimJobRun("mirror-schedule:a", time.Hour); err != nil || !claimed { - t.Fatalf("ClaimJobRun after migration = %v, %v; want claimed", claimed, err) - } - }) -} - -func TestFetchLeases(t *testing.T) { - db := createTestPostgresDB(t) - if db == nil { - t.Skip("PROXY_DATABASE_URL not set") - } - defer func() { _ = db.Close() }() - - acquire := func(owner string, ttl time.Duration) bool { - t.Helper() - ok, err := db.AcquireFetchLease("k", owner, ttl) - if err != nil { - t.Fatalf("AcquireFetchLease(%s): %v", owner, err) - } - return ok - } - if !acquire("a", time.Minute) { - t.Fatal("a could not take a free lease") - } - if acquire("b", time.Minute) { - t.Fatal("b took a lease a holds") - } - if held, err := db.RenewFetchLease("k", "b", time.Minute); err != nil || held { - t.Fatalf("b renewed a's lease: %v, %v", held, err) - } - if held, err := db.RenewFetchLease("k", "a", time.Minute); err != nil || !held { - t.Fatalf("a could not renew its lease: %v, %v", held, err) - } - // Only the owner can release it. - if err := db.ReleaseFetchLease("k", "b"); err != nil { - t.Fatal(err) - } - if acquire("b", time.Minute) { - t.Fatal("b's release dropped a's lease") - } - if err := db.ReleaseFetchLease("k", "a"); err != nil { - t.Fatal(err) - } - if !acquire("b", time.Millisecond) { - t.Fatal("b could not take the lease a released") - } - time.Sleep(20 * time.Millisecond) - if !acquire("c", time.Millisecond) { - t.Fatal("c could not take b's expired lease") - } - time.Sleep(20 * time.Millisecond) - if n, err := db.DeleteExpiredFetchLeases(0); err != nil || n != 1 { - t.Fatalf("DeleteExpiredFetchLeases = %d, %v; want 1", n, err) - } -} - -func TestFetchLeaseFailure(t *testing.T) { - db := createTestPostgresDB(t) - if db == nil { - t.Skip("PROXY_DATABASE_URL not set") - } - defer func() { _ = db.Close() }() - - if ok, err := db.AcquireFetchLease("k", "a", time.Minute); err != nil || !ok { - t.Fatalf("a could not take a free lease: %v, %v", ok, err) - } - if owner, failed, ok, err := db.FetchLeaseHolder("k"); err != nil || !ok || owner != "a" || failed { - t.Fatalf("FetchLeaseHolder = %q, failed %v, ok %v, %v; want a's live lease", owner, failed, ok, err) - } - // A failed lease stays readable for those waiting on it, and is free to - // take. - if err := db.FailFetchLease("k", "b"); err != nil { - t.Fatal(err) - } - if _, failed, _, _ := db.FetchLeaseHolder("k"); failed { - t.Fatal("b failed a's lease") - } - if err := db.FailFetchLease("k", "a"); err != nil { - t.Fatal(err) - } - if owner, failed, ok, err := db.FetchLeaseHolder("k"); err != nil || !ok || owner != "a" || !failed { - t.Fatalf("FetchLeaseHolder = %q, failed %v, ok %v, %v; want a's failed lease", owner, failed, ok, err) - } - if ok, err := db.AcquireFetchLease("k", "b", time.Minute); err != nil || !ok { - t.Fatalf("b could not take a's failed lease: %v, %v", ok, err) - } - if owner, failed, _, _ := db.FetchLeaseHolder("k"); owner != "b" || failed { - t.Fatalf("after takeover: owner %q, failed %v", owner, failed) - } - if err := db.ReleaseFetchLease("k", "b"); err != nil { - t.Fatal(err) - } - if _, _, ok, err := db.FetchLeaseHolder("k"); err != nil || ok { - t.Fatalf("FetchLeaseHolder after release: ok %v, %v", ok, err) - } -} diff --git a/internal/database/metadata_cache_test.go b/internal/database/metadata_cache_test.go index ee16c25..5701816 100644 --- a/internal/database/metadata_cache_test.go +++ b/internal/database/metadata_cache_test.go @@ -2,9 +2,7 @@ package database import ( "database/sql" - "fmt" "path/filepath" - "slices" "testing" "time" ) @@ -31,14 +29,9 @@ func TestUpsertAndGetMetadataCache(t *testing.T) { Name: "lodash", StoragePath: "_metadata/npm/lodash/metadata", ETag: sql.NullString{String: `"abc123"`, Valid: true}, - Link: sql.NullString{String: `; rel="next"`, Valid: true}, ContentType: sql.NullString{String: "application/json", Valid: true}, - ContentDigest: sql.NullString{ - String: "sha256:0123456789abcdef", - Valid: true, - }, - Size: sql.NullInt64{Int64: 1024, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + Size: sql.NullInt64{Int64: 1024, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, } err := db.UpsertMetadataCache(entry) @@ -66,15 +59,9 @@ func TestUpsertAndGetMetadataCache(t *testing.T) { if !got.ETag.Valid || got.ETag.String != `"abc123"` { t.Errorf("etag = %v, want %q", got.ETag, `"abc123"`) } - if !got.Link.Valid || got.Link.String != `; rel="next"` { - t.Errorf("link = %v, want next link", got.Link) - } if !got.ContentType.Valid || got.ContentType.String != "application/json" { t.Errorf("content_type = %v, want %q", got.ContentType, "application/json") } - if !got.ContentDigest.Valid || got.ContentDigest.String != "sha256:0123456789abcdef" { - t.Errorf("content_digest = %v, want %q", got.ContentDigest, "sha256:0123456789abcdef") - } if !got.Size.Valid || got.Size.Int64 != 1024 { t.Errorf("size = %v, want 1024", got.Size) } @@ -164,9 +151,6 @@ func TestUpsertMetadataCacheNullableFields(t *testing.T) { if got.ContentType.Valid { t.Error("expected null content_type") } - if got.Link.Valid { - t.Error("expected null link") - } if got.Size.Valid { t.Error("expected null size") } @@ -194,347 +178,3 @@ func TestMetadataCacheTableCreatedByMigration(t *testing.T) { t.Error("metadata_cache table should exist after migration") } } - -func TestMetadataCacheContentDigestMigrationPreservesExistingRows(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "test.db") - db, err := Create(dbPath) - if err != nil { - t.Fatalf("Create failed: %v", err) - } - defer func() { _ = db.Close() }() - - if _, err := db.Exec("ALTER TABLE metadata_cache DROP COLUMN content_digest"); err != nil { - t.Fatalf("dropping content_digest: %v", err) - } - if _, err := db.Exec("DELETE FROM migrations WHERE name = ?", "006_add_metadata_content_digest"); err != nil { - t.Fatalf("resetting digest migration: %v", err) - } - if _, err := db.Exec(` - INSERT INTO metadata_cache (ecosystem, name, storage_path, content_type, size, fetched_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - `, "oci-manifest", "cache-key", "_metadata/oci-manifest/cache-key/metadata", "application/json", 2, time.Now(), time.Now(), time.Now()); err != nil { - t.Fatalf("inserting legacy cache row: %v", err) - } - - if err := db.MigrateSchema(); err != nil { - t.Fatalf("MigrateSchema() error = %v", err) - } - hasDigest, err := db.HasColumn("metadata_cache", "content_digest") - if err != nil { - t.Fatalf("HasColumn() error = %v", err) - } - if !hasDigest { - t.Fatal("metadata_cache.content_digest was not added") - } - - entry, err := db.GetMetadataCache("oci-manifest", "cache-key") - if err != nil { - t.Fatalf("GetMetadataCache() error = %v", err) - } - if entry == nil || entry.StoragePath != "_metadata/oci-manifest/cache-key/metadata" { - t.Fatalf("existing metadata cache row was not preserved: %#v", entry) - } - if entry.ContentDigest.Valid { - t.Errorf("legacy content digest = %q, want NULL", entry.ContentDigest.String) - } -} - -func TestMetadataCacheLinkMigrationPreservesExistingRows(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "test.db") - db, err := Create(dbPath) - if err != nil { - t.Fatalf("Create failed: %v", err) - } - defer func() { _ = db.Close() }() - - if _, err := db.Exec("ALTER TABLE metadata_cache DROP COLUMN link"); err != nil { - t.Fatalf("dropping link: %v", err) - } - if _, err := db.Exec("DELETE FROM migrations WHERE name = ?", "007_add_metadata_link"); err != nil { - t.Fatalf("resetting link migration: %v", err) - } - if _, err := db.Exec(` - INSERT INTO metadata_cache (ecosystem, name, storage_path, content_type, size, fetched_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - `, "oci-tags", "cache-key", "_metadata/oci-tags/cache-key/metadata", "application/json", 2, time.Now(), time.Now(), time.Now()); err != nil { - t.Fatalf("inserting legacy cache row: %v", err) - } - - if err := db.MigrateSchema(); err != nil { - t.Fatalf("MigrateSchema() error = %v", err) - } - hasLink, err := db.HasColumn("metadata_cache", "link") - if err != nil { - t.Fatalf("HasColumn() error = %v", err) - } - if !hasLink { - t.Fatal("metadata_cache.link was not added") - } - - entry, err := db.GetMetadataCache("oci-tags", "cache-key") - if err != nil { - t.Fatalf("GetMetadataCache() error = %v", err) - } - if entry == nil || entry.StoragePath != "_metadata/oci-tags/cache-key/metadata" { - t.Fatalf("existing metadata cache row was not preserved: %#v", entry) - } - if entry.Link.Valid { - t.Errorf("legacy link = %q, want NULL", entry.Link.String) - } -} - -func TestMetadataCacheContentEncodingMigrationPreservesExistingRows(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "test.db") - db, err := Create(dbPath) - if err != nil { - t.Fatalf("Create failed: %v", err) - } - defer func() { _ = db.Close() }() - - if _, err := db.Exec("ALTER TABLE metadata_cache DROP COLUMN content_encoding"); err != nil { - t.Fatalf("dropping content_encoding: %v", err) - } - if _, err := db.Exec("DELETE FROM migrations WHERE name = ?", "008_add_metadata_content_encoding"); err != nil { - t.Fatalf("resetting content_encoding migration: %v", err) - } - if _, err := db.Exec(` - INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type, size, fetched_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - `, "apk", "cache-key", "_metadata/apk/cache-key/metadata", "\"legacy-etag\"", "application/octet-stream", 2, time.Now(), time.Now(), time.Now()); err != nil { - t.Fatalf("inserting legacy cache row: %v", err) - } - - if err := db.MigrateSchema(); err != nil { - t.Fatalf("MigrateSchema() error = %v", err) - } - hasEncoding, err := db.HasColumn("metadata_cache", "content_encoding") - if err != nil { - t.Fatalf("HasColumn() error = %v", err) - } - if !hasEncoding { - t.Fatal("metadata_cache.content_encoding was not added") - } - - entry, err := db.GetMetadataCache("apk", "cache-key") - if err != nil { - t.Fatalf("GetMetadataCache() error = %v", err) - } - if entry == nil || entry.StoragePath != "_metadata/apk/cache-key/metadata" { - t.Fatalf("existing metadata cache row was not preserved: %#v", entry) - } - if entry.ContentEncoding.Valid { - t.Errorf("legacy content encoding = %q, want NULL", entry.ContentEncoding.String) - } - // The migration must clear the pre-fix validators so the row is refetched - // once with identity instead of a 304 re-serving the decompressed copy. - if entry.ETag.Valid { - t.Errorf("legacy etag = %q, want cleared", entry.ETag.String) - } - if entry.FetchedAt.Valid { - t.Errorf("legacy fetched_at = %v, want cleared", entry.FetchedAt.Time) - } -} - -func TestMetadataCacheRoundTripsContentEncoding(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "test.db") - db, err := Create(dbPath) - if err != nil { - t.Fatalf("Create failed: %v", err) - } - defer func() { _ = db.Close() }() - - entry := &MetadataCacheEntry{ - Ecosystem: "apk", - Name: "index-key", - StoragePath: "_metadata/apk/index-key/metadata", - ContentType: sql.NullString{String: "application/octet-stream", Valid: true}, - ContentEncoding: sql.NullString{String: "gzip", Valid: true}, - Size: sql.NullInt64{Int64: 10, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - } - if err := db.UpsertMetadataCache(entry); err != nil { - t.Fatalf("UpsertMetadataCache() error = %v", err) - } - - got, err := db.GetMetadataCache("apk", "index-key") - if err != nil { - t.Fatalf("GetMetadataCache() error = %v", err) - } - if got == nil || !got.ContentEncoding.Valid || got.ContentEncoding.String != "gzip" { - t.Fatalf("content encoding round-trip failed: %#v", got) - } -} - -// On Postgres, fetched_at is a TIMESTAMP column without a zone. East of UTC -// it must still read back as the moment it was written, or metadata_ttl -// expires hours late. last_modified is an HTTP date in UTC and must keep it. -func TestMetadataCacheTimesOffUTC(t *testing.T) { - tokyo, err := time.LoadLocation("Asia/Tokyo") - if err != nil { - t.Skipf("time zone data unavailable: %v", err) - } - prev := time.Local - time.Local = tokyo - t.Cleanup(func() { time.Local = prev }) - - runWithBothDatabases(t, func(t *testing.T, db *DB) { - fetched := time.Now().Add(-90 * time.Minute) - modified := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) - entry := &MetadataCacheEntry{ - Ecosystem: testEcosystemNPM, - Name: "tz", - StoragePath: "_metadata/npm/tz/metadata", - LastModified: sql.NullTime{Time: modified, Valid: true}, - FetchedAt: sql.NullTime{Time: fetched, Valid: true}, - } - if err := db.UpsertMetadataCache(entry); err != nil { - t.Fatalf("UpsertMetadataCache() error = %v", err) - } - - got, err := db.GetMetadataCache(testEcosystemNPM, "tz") - if err != nil || got == nil { - t.Fatalf("GetMetadataCache() = %v, %v", got, err) - } - if diff := got.FetchedAt.Time.Sub(fetched).Abs(); diff > time.Second { - t.Errorf("fetched_at read back as %v, want %v", got.FetchedAt.Time, fetched) - } - if !got.LastModified.Time.Equal(modified) { - t.Errorf("last_modified read back as %v, want %v", got.LastModified.Time, modified) - } - }) -} - -// TestRefreshMetadataCache checks that a refresh moves only fetched_at on, and -// only while the row still records the digest it was asked for. -func TestRefreshMetadataCache(t *testing.T) { - db := setupMetadataCacheDB(t) - fetched := time.Now().Add(-time.Hour).UTC().Truncate(time.Second) - entry := &MetadataCacheEntry{ - Ecosystem: testEcosystemNPM, - Name: "lodash", - StoragePath: "_metadata/npm/lodash/metadata", - ETag: sql.NullString{String: `"abc123"`, Valid: true}, - ContentDigest: sql.NullString{String: "sha256:aaaa", Valid: true}, - Size: sql.NullInt64{Int64: 1024, Valid: true}, - FetchedAt: sql.NullTime{Time: fetched, Valid: true}, - } - if err := db.UpsertMetadataCache(entry); err != nil { - t.Fatal(err) - } - - refreshed, err := db.RefreshMetadataCache(testEcosystemNPM, "lodash", "sha256:bbbb", time.Now()) - if err != nil || refreshed { - t.Fatalf("refresh with another digest = %v, %v, want false", refreshed, err) - } - if got, _ := db.GetMetadataCache(testEcosystemNPM, "lodash"); !got.FetchedAt.Time.Equal(fetched) { - t.Errorf("fetched_at = %v after a refresh with another digest, want %v", got.FetchedAt.Time, fetched) - } - - now := time.Now().Truncate(time.Second) - refreshed, err = db.RefreshMetadataCache(testEcosystemNPM, "lodash", "sha256:aaaa", now) - if err != nil || !refreshed { - t.Fatalf("refresh = %v, %v, want true", refreshed, err) - } - got, err := db.GetMetadataCache(testEcosystemNPM, "lodash") - if err != nil { - t.Fatal(err) - } - if !got.FetchedAt.Time.Equal(now) { - t.Errorf("fetched_at = %v, want %v", got.FetchedAt.Time, now) - } - if got.ETag.String != `"abc123"` || got.ContentDigest.String != "sha256:aaaa" || got.Size.Int64 != 1024 || got.StoragePath != entry.StoragePath { - t.Errorf("refresh changed the row: %+v", got) - } - - if refreshed, err := db.RefreshMetadataCache(testEcosystemNPM, "missing", "sha256:aaaa", now); err != nil || refreshed { - t.Errorf("refresh of a missing row = %v, %v, want false", refreshed, err) - } -} - -// TestExpireMetadataCache checks that expiring clears fetched_at only on the -// named rows of the ecosystem fetched before the cutoff, and keeps their -// documents and validators. -func TestExpireMetadataCache(t *testing.T) { - db := setupMetadataCacheDB(t) - cutoff := time.Now().UTC().Truncate(time.Second) - rows := []struct { - ecosystem, name string - fetched time.Time - }{ - {"composer", "acme/lib", cutoff.Add(-time.Minute)}, - {"composer", "acme/lib~dev", cutoff.Add(-time.Minute)}, - {"composer", "acme/new", cutoff.Add(time.Second)}, - {"composer", "acme/other", cutoff.Add(-time.Minute)}, - {"composer", "routes/abc/acme/lib", cutoff.Add(-time.Minute)}, - {testEcosystemNPM, "acme/lib", cutoff.Add(-time.Minute)}, - } - for _, r := range rows { - err := db.UpsertMetadataCache(&MetadataCacheEntry{ - Ecosystem: r.ecosystem, - Name: r.name, - StoragePath: "_metadata/" + r.ecosystem + "/" + r.name, - ETag: sql.NullString{String: `"abc"`, Valid: true}, - ContentDigest: sql.NullString{String: "sha256:aaaa", Valid: true}, - FetchedAt: sql.NullTime{Time: r.fetched, Valid: true}, - }) - if err != nil { - t.Fatal(err) - } - } - - expired, err := db.ExpireMetadataCache("composer", []string{"acme/lib", "acme/lib~dev", "acme/new", "acme/missing"}, cutoff) - if err != nil { - t.Fatal(err) - } - slices.Sort(expired) - if want := []string{"acme/lib", "acme/lib~dev"}; !slices.Equal(expired, want) { - t.Errorf("expired = %v, want %v", expired, want) - } - got, err := db.GetMetadataCache("composer", "acme/lib") - if err != nil { - t.Fatal(err) - } - if got.FetchedAt.Valid || got.ETag.String != `"abc"` || got.ContentDigest.String != "sha256:aaaa" { - t.Errorf("expired row = %+v, want no fetched_at and the same validators", got) - } - for _, r := range []struct{ ecosystem, name string }{{"composer", "acme/new"}, {"composer", "acme/other"}, {testEcosystemNPM, "acme/lib"}} { - if got, _ := db.GetMetadataCache(r.ecosystem, r.name); !got.FetchedAt.Valid { - t.Errorf("%s %s expired, want it left alone", r.ecosystem, r.name) - } - } - - expired, err = db.ExpireAllMetadataCache("composer", cutoff) - if err != nil { - t.Fatal(err) - } - if want := []string{"acme/other"}; !slices.Equal(expired, want) { - t.Errorf("expired all = %v, want %v (routed and newer rows kept)", expired, want) - } -} - -// TestExpireMetadataCacheBatches checks that more names than one statement -// takes are all expired. -func TestExpireMetadataCacheBatches(t *testing.T) { - db := setupMetadataCacheDB(t) - cutoff := time.Now().UTC() - names := make([]string, expireMetadataBatch+2) - for i := range names { - names[i] = fmt.Sprintf("pkg-%d", i) - } - for _, name := range []string{names[0], names[len(names)-1]} { - err := db.UpsertMetadataCache(&MetadataCacheEntry{ - Ecosystem: testEcosystemNPM, Name: name, StoragePath: "_metadata/npm/" + name, - FetchedAt: sql.NullTime{Time: cutoff.Add(-time.Minute), Valid: true}, - }) - if err != nil { - t.Fatal(err) - } - } - expired, err := db.ExpireMetadataCache(testEcosystemNPM, names, cutoff) - if err != nil { - t.Fatal(err) - } - if len(expired) != 2 { - t.Errorf("expired = %v, want both stored names", expired) - } -} diff --git a/internal/database/metadata_dates.go b/internal/database/metadata_dates.go deleted file mode 100644 index f705f29..0000000 --- a/internal/database/metadata_dates.go +++ /dev/null @@ -1,77 +0,0 @@ -package database - -import ( - "database/sql" - "errors" - "fmt" - "time" -) - -// MetadataDate is the Last-Modified a package's rewritten metadata is served -// with, and the ETag of the rewrite it dates. -// -// Rewritten metadata has no date of its own, so the proxy assigns one when a -// package starts serving a new rewrite. Keeping it in the database rather than -// in process memory is what lets replicas sharing the database agree on it: a -// date one replica gave out is never given by another to a different rewrite. -type MetadataDate struct { - ETag string - LastModified time.Time -} - -type metadataDateRow struct { - ETag string `db:"etag"` - LastModified int64 `db:"last_modified"` -} - -// GetMetadataDate returns the date recorded for a package's metadata as -// rewritten for proxyURL, or nil when there is none. -func (db *DB) GetMetadataDate(ecosystem, proxyURL, name string) (*MetadataDate, error) { - var row metadataDateRow - err := db.Get(&row, db.Rebind(` - SELECT etag, last_modified FROM metadata_dates - WHERE ecosystem = ? AND proxy_url = ? AND name = ? - `), ecosystem, proxyURL, name) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("getting metadata date: %w", err) - } - return &MetadataDate{ETag: row.ETag, LastModified: time.Unix(row.LastModified, 0).UTC()}, nil -} - -// InsertMetadataDate records the first date of a package's metadata and -// reports whether it did; it leaves a date another replica recorded first. -func (db *DB) InsertMetadataDate(ecosystem, proxyURL, name string, date MetadataDate) (bool, error) { - result, err := db.Exec(db.Rebind(` - INSERT INTO metadata_dates (ecosystem, proxy_url, name, etag, last_modified) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT (ecosystem, proxy_url, name) DO NOTHING - `), ecosystem, proxyURL, name, date.ETag, date.LastModified.Unix()) - if err != nil { - return false, fmt.Errorf("inserting metadata date: %w", err) - } - return affectedOne(result) -} - -// SwapMetadataDate replaces a package's date with next if it is still old, -// and reports whether it did; it leaves a date another replica swapped in. -func (db *DB) SwapMetadataDate(ecosystem, proxyURL, name string, old, next MetadataDate) (bool, error) { - result, err := db.Exec(db.Rebind(` - UPDATE metadata_dates SET etag = ?, last_modified = ? - WHERE ecosystem = ? AND proxy_url = ? AND name = ? AND etag = ? AND last_modified = ? - `), next.ETag, next.LastModified.Unix(), ecosystem, proxyURL, name, old.ETag, old.LastModified.Unix()) - if err != nil { - return false, fmt.Errorf("swapping metadata date: %w", err) - } - return affectedOne(result) -} - -func affectedOne(result sql.Result) (bool, error) { - n, err := result.RowsAffected() - if err != nil { - return false, fmt.Errorf("counting affected rows: %w", err) - } - return n == 1, nil -} diff --git a/internal/database/metadata_dates_test.go b/internal/database/metadata_dates_test.go deleted file mode 100644 index 66c178a..0000000 --- a/internal/database/metadata_dates_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package database - -import ( - "database/sql" - "strings" - "testing" - "time" -) - -const testProxyURL = "http://proxy.example" - -func TestMetadataDateMissing(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "missing") - if err != nil || got != nil { - t.Errorf("GetMetadataDate() = %+v, %v; want nil, nil", got, err) - } - }) -} - -func TestMetadataDateInsertOnce(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - first := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)} - second := MetadataDate{ETag: `"b"`, LastModified: first.LastModified.Add(time.Second)} - - inserted, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", first) - if err != nil || !inserted { - t.Fatalf("first InsertMetadataDate() = %v, %v; want true", inserted, err) - } - inserted, err = db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", second) - if err != nil || inserted { - t.Fatalf("second InsertMetadataDate() = %v, %v; want false, the row exists", inserted, err) - } - - got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "left-pad") - if err != nil || got == nil || *got != first { - t.Errorf("GetMetadataDate() = %+v, %v; want %+v", got, err, first) - } - }) -} - -// A package's date is kept per proxy URL: each one rewrites the document to -// different download URLs, so each serves a different representation. -func TestMetadataDateKeyedByProxyURL(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - date := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)} - if _, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", date); err != nil { - t.Fatal(err) - } - inserted, err := db.InsertMetadataDate(testEcosystemNPM, "http://other.example", "left-pad", date) - if err != nil || !inserted { - t.Errorf("InsertMetadataDate() for another proxy URL = %v, %v; want true", inserted, err) - } - }) -} - -// SwapMetadataDate is a compare-and-swap: it only replaces the date it was -// given, so of two replicas swapping from the same date only one succeeds. -func TestMetadataDateSwap(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - a := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)} - b := MetadataDate{ETag: `"b"`, LastModified: a.LastModified.Add(time.Second)} - c := MetadataDate{ETag: `"c"`, LastModified: b.LastModified.Add(time.Second)} - if _, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", a); err != nil { - t.Fatal(err) - } - - swapped, err := db.SwapMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", a, b) - if err != nil || !swapped { - t.Fatalf("SwapMetadataDate(a, b) = %v, %v; want true", swapped, err) - } - swapped, err = db.SwapMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", a, c) - if err != nil || swapped { - t.Fatalf("SwapMetadataDate(a, c) after a was replaced = %v, %v; want false", swapped, err) - } - - got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "left-pad") - if err != nil || got == nil || *got != b { - t.Errorf("GetMetadataDate() = %+v, %v; want %+v", got, err, b) - } - }) -} - -// Dates are whole seconds in UTC whatever the local zone, so every replica -// reads back the date another one wrote. -func TestMetadataDateIsUTCSeconds(t *testing.T) { - tokyo, err := time.LoadLocation("Asia/Tokyo") - if err != nil { - t.Skipf("time zone data unavailable: %v", err) - } - prev := time.Local - time.Local = tokyo - t.Cleanup(func() { time.Local = prev }) - - runWithBothDatabases(t, func(t *testing.T, db *DB) { - written := time.Date(2026, 10, 10, 21, 0, 1, 700_000_000, tokyo) - if _, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "tz", MetadataDate{ETag: `"a"`, LastModified: written}); err != nil { - t.Fatal(err) - } - got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "tz") - if err != nil || got == nil { - t.Fatalf("GetMetadataDate() = %+v, %v", got, err) - } - if want := time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC); !got.LastModified.Equal(want) || got.LastModified.Location() != time.UTC { - t.Errorf("LastModified read back as %v, want %v", got.LastModified, want) - } - }) -} - -// TestMetadataDatesMigrationKeepsCachedMetadata upgrades a database from -// before migration 011 that holds cached npm and Composer metadata, and checks -// that the cached rows come through unchanged and dates can be recorded. -func TestMetadataDatesMigrationKeepsCachedMetadata(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - rollBackMetadataDates(t, db) - seeded := map[string]string{testEcosystemNPM: "left-pad", "composer": "vendor/pkg"} - before := make(map[string]*MetadataCacheEntry, len(seeded)) - for ecosystem, name := range seeded { - before[ecosystem] = seedMetadataCache(t, db, ecosystem, name) - } - - if err := db.MigrateSchema(); err != nil { - t.Fatalf("MigrateSchema() error = %v", err) - } - - for ecosystem, name := range seeded { - after, err := db.GetMetadataCache(ecosystem, name) - if err != nil || after == nil || *after != *before[ecosystem] { - t.Errorf("metadata row %s/%s after migration = %+v, %v; want %+v", ecosystem, name, after, err, before[ecosystem]) - } - date := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)} - if inserted, err := db.InsertMetadataDate(ecosystem, testProxyURL, name, date); err != nil || !inserted { - t.Errorf("InsertMetadataDate(%s/%s) after migration = %v, %v; want true", ecosystem, name, inserted, err) - } - } - }) -} - -// seedMetadataCache stores a metadata row as the metadata cache writes one -// and returns it as read back. -func seedMetadataCache(t *testing.T, db *DB, ecosystem, name string) *MetadataCacheEntry { - t.Helper() - err := db.UpsertMetadataCache(&MetadataCacheEntry{ - Ecosystem: ecosystem, - Name: name, - StoragePath: "_metadata/" + ecosystem + "/" + name + "/metadata", - ETag: sql.NullString{String: `"upstream"`, Valid: true}, - ContentType: sql.NullString{String: "application/json", Valid: true}, - ContentDigest: sql.NullString{String: "sha256:" + strings.Repeat("ab", 32), Valid: true}, - Size: sql.NullInt64{Int64: 123, Valid: true}, - LastModified: sql.NullTime{Time: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now().UTC().Truncate(time.Second), Valid: true}, - }) - if err != nil { - t.Fatal(err) - } - entry, err := db.GetMetadataCache(ecosystem, name) - if err != nil || entry == nil { - t.Fatalf("GetMetadataCache(%s/%s) = %v, %v", ecosystem, name, entry, err) - } - return entry -} - -// rollBackMetadataDates returns db to the schema from before migration 011, -// which only added the metadata_dates table. -func rollBackMetadataDates(t *testing.T, db *DB) { - t.Helper() - if _, err := db.Exec("DROP TABLE metadata_dates"); err != nil { - t.Fatal(err) - } - if _, err := db.Exec("DELETE FROM migrations WHERE name = '011_add_metadata_dates'"); err != nil { - t.Fatal(err) - } - if has, err := db.HasTable("metadata_dates"); err != nil || has { - t.Fatalf("HasTable(metadata_dates) after rollback = %v, %v; want false", has, err) - } -} diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go deleted file mode 100644 index 268baf7..0000000 --- a/internal/database/metadata_snapshots.go +++ /dev/null @@ -1,156 +0,0 @@ -package database - -import ( - "database/sql" - "fmt" - "math" - "time" -) - -// MetadataSnapshot is one kept version of a metadata document, so a request -// for a past date can be answered with the document that was current then. -// It is valid from ValidFromMS (Unix milliseconds, the time the proxy first -// stored it) until the next snapshot of the same ecosystem and name. -type MetadataSnapshot struct { - ID int64 `db:"id"` - Ecosystem string `db:"ecosystem"` - Name string `db:"name"` - StoragePath string `db:"storage_path"` - ContentType sql.NullString `db:"content_type"` - ContentEncoding sql.NullString `db:"content_encoding"` - ContentDigest string `db:"content_digest"` - Size int64 `db:"size"` - ValidFromMS int64 `db:"valid_from_ms"` -} - -// ValidFrom returns the time the snapshot became current. -func (s *MetadataSnapshot) ValidFrom() time.Time { - return time.UnixMilli(s.ValidFromMS) -} - -const metadataSnapshotColumns = `id, ecosystem, name, storage_path, content_type, content_encoding, - content_digest, size, valid_from_ms` - -// MetadataSnapshotAt returns the snapshot of ecosystem and name that was -// current at at: the newest one that became current before it. It returns nil -// when there is none. -func (db *DB) MetadataSnapshotAt(ecosystem, name string, at time.Time) (*MetadataSnapshot, error) { - return db.metadataSnapshotBefore(ecosystem, name, at.UnixMilli()) -} - -// LatestMetadataSnapshot returns the newest snapshot of ecosystem and name, -// or nil when there is none. -func (db *DB) LatestMetadataSnapshot(ecosystem, name string) (*MetadataSnapshot, error) { - return db.metadataSnapshotBefore(ecosystem, name, math.MaxInt64) -} - -func (db *DB) metadataSnapshotBefore(ecosystem, name string, beforeMS int64) (*MetadataSnapshot, error) { - var snap MetadataSnapshot - err := db.Get(&snap, db.Rebind(`SELECT `+metadataSnapshotColumns+` - FROM metadata_snapshots - WHERE ecosystem = ? AND name = ? AND valid_from_ms < ? - ORDER BY valid_from_ms DESC, id DESC - LIMIT 1`), ecosystem, name, beforeMS) - if err == sql.ErrNoRows { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("reading metadata snapshot: %w", err) - } - return &snap, nil -} - -// AddMetadataSnapshot records snap as the version of its document current -// from snap.ValidFromMS, unless the snapshot current at that time already has -// its digest, and reports whether it did. On Postgres it first takes an -// advisory lock on the document, so two replicas can't both read the previous -// digest and both insert. The casts let Postgres type the parameters of the -// INSERT ... SELECT. -func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { - tx, err := db.Beginx() - if err != nil { - return false, fmt.Errorf("recording metadata snapshot: %w", err) - } - defer func() { _ = tx.Rollback() }() - if db.dialect == DialectPostgres { - if _, err := tx.Exec("SELECT pg_advisory_xact_lock($1)", metadataSnapshotLockKey(snap.Ecosystem, snap.Name)); err != nil { - return false, fmt.Errorf("locking metadata snapshots: %w", err) - } - } - res, err := tx.Exec(db.Rebind(` - INSERT INTO metadata_snapshots (ecosystem, name, storage_path, content_type, content_encoding, - content_digest, size, valid_from_ms) - SELECT CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), - CAST(? AS TEXT), CAST(? AS BIGINT), CAST(? AS BIGINT) - WHERE COALESCE(( - SELECT content_digest FROM metadata_snapshots - WHERE ecosystem = ? AND name = ? AND valid_from_ms <= CAST(? AS BIGINT) - ORDER BY valid_from_ms DESC, id DESC - LIMIT 1 - ), '') <> CAST(? AS TEXT) - `), snap.Ecosystem, snap.Name, snap.StoragePath, snap.ContentType, snap.ContentEncoding, - snap.ContentDigest, snap.Size, snap.ValidFromMS, - snap.Ecosystem, snap.Name, snap.ValidFromMS, snap.ContentDigest) - if err != nil { - return false, fmt.Errorf("recording metadata snapshot: %w", err) - } - n, err := res.RowsAffected() - if err != nil { - return false, err - } - if err := tx.Commit(); err != nil { - return false, fmt.Errorf("recording metadata snapshot: %w", err) - } - return n > 0, nil -} - -// metadataSnapshotLockKey is the advisory lock AddMetadataSnapshot takes for -// a document. -func metadataSnapshotLockKey(ecosystem, name string) int64 { - return lockKey("metadata-snapshot:" + ecosystem + "\x00" + name) -} - -// PruneMetadataSnapshots deletes the snapshots of ecosystem that stopped -// being current before cutoff, that is those replaced by a newer snapshot -// that became current before cutoff, and queues their blobs for deletion. The -// newest snapshot of each document is always kept. It returns the number of -// snapshots deleted. -func (db *DB) PruneMetadataSnapshots(ecosystem string, cutoff time.Time) (int, error) { - tx, err := db.Beginx() - if err != nil { - return 0, err - } - defer func() { _ = tx.Rollback() }() - - var expired []struct { - ID int64 `db:"id"` - StoragePath string `db:"storage_path"` - } - err = tx.Select(&expired, db.Rebind(` - SELECT s.id, s.storage_path FROM metadata_snapshots s - WHERE s.ecosystem = ? - AND EXISTS ( - SELECT 1 FROM metadata_snapshots n - WHERE n.ecosystem = s.ecosystem AND n.name = s.name - AND (n.valid_from_ms > s.valid_from_ms OR (n.valid_from_ms = s.valid_from_ms AND n.id > s.id)) - AND n.valid_from_ms < ? - ) - `), ecosystem, cutoff.UnixMilli()) - if err != nil { - return 0, fmt.Errorf("listing expired metadata snapshots: %w", err) - } - - queuedAt := time.Now().UTC() - for _, row := range expired { - if _, err := tx.Exec(db.Rebind(`DELETE FROM metadata_snapshots WHERE id = ?`), row.ID); err != nil { - return 0, fmt.Errorf("deleting metadata snapshot: %w", err) - } - if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), row.StoragePath, queuedAt); err != nil { - return 0, fmt.Errorf("queueing metadata snapshot blob: %w", err) - } - } - if err := tx.Commit(); err != nil { - return 0, err - } - return len(expired), nil -} diff --git a/internal/database/metadata_snapshots_test.go b/internal/database/metadata_snapshots_test.go deleted file mode 100644 index 71865b9..0000000 --- a/internal/database/metadata_snapshots_test.go +++ /dev/null @@ -1,190 +0,0 @@ -package database - -import ( - "testing" - "time" -) - -func addSnapshot(t *testing.T, db *DB, name, digest string, validFrom time.Time) bool { - t.Helper() - added, err := db.AddMetadataSnapshot(&MetadataSnapshot{ - Ecosystem: "alpine", - Name: name, - StoragePath: "_snapshots/alpine/" + name + "/" + digest, - ContentDigest: digest, - Size: 1, - ValidFromMS: validFrom.UnixMilli(), - }) - if err != nil { - t.Fatalf("AddMetadataSnapshot: %v", err) - } - return added -} - -func snapshotDigestAt(t *testing.T, db *DB, name string, at time.Time) string { - t.Helper() - snap, err := db.MetadataSnapshotAt("alpine", name, at) - if err != nil { - t.Fatalf("MetadataSnapshotAt: %v", err) - } - if snap == nil { - return "" - } - return snap.ContentDigest -} - -func TestMetadataSnapshots_AddAndLookup(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - day := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) - if !addSnapshot(t, db, "idx", "sha256:a", day) { - t.Fatal("first snapshot not added") - } - if addSnapshot(t, db, "idx", "sha256:a", day.Add(time.Hour)) { - t.Error("unchanged digest added again") - } - if !addSnapshot(t, db, "idx", "sha256:b", day.Add(48*time.Hour)) { - t.Fatal("changed digest not added") - } - // A rollback to an earlier version is a new snapshot. - if !addSnapshot(t, db, "idx", "sha256:a", day.Add(96*time.Hour)) { - t.Fatal("rolled back digest not added") - } - - for _, tc := range []struct { - at time.Time - want string - }{ - {day, ""}, - {day.Add(time.Minute), "sha256:a"}, - {day.Add(72 * time.Hour), "sha256:b"}, - {day.Add(100 * time.Hour), "sha256:a"}, - } { - if got := snapshotDigestAt(t, db, "idx", tc.at); got != tc.want { - t.Errorf("snapshot at %s = %q, want %q", tc.at, got, tc.want) - } - } - latest, err := db.LatestMetadataSnapshot("alpine", "idx") - if err != nil || latest == nil || !latest.ValidFrom().Equal(day.Add(96*time.Hour)) { - t.Errorf("LatestMetadataSnapshot = %+v, %v", latest, err) - } - }) -} - -// TestMetadataSnapshots_RecordedLate adds a version after a newer one was -// recorded, as a request that fetched it first but recorded it last does. It -// is compared with the version current at its own time, so it keeps its place -// in the history. -func TestMetadataSnapshots_RecordedLate(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - day := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) - addSnapshot(t, db, "idx", "sha256:a", day) - addSnapshot(t, db, "idx", "sha256:c", day.Add(2*time.Hour)) - if !addSnapshot(t, db, "idx", "sha256:b", day.Add(time.Hour)) { - t.Fatal("version recorded after a newer one not added") - } - if addSnapshot(t, db, "idx", "sha256:b", day.Add(90*time.Minute)) { - t.Error("version already current at that time added again") - } - for _, tc := range []struct { - at time.Time - want string - }{ - {day.Add(30 * time.Minute), "sha256:a"}, - {day.Add(100 * time.Minute), "sha256:b"}, - {day.Add(3 * time.Hour), "sha256:c"}, - } { - if got := snapshotDigestAt(t, db, "idx", tc.at); got != tc.want { - t.Errorf("snapshot at %s = %q, want %q", tc.at, got, tc.want) - } - } - }) -} - -func TestMetadataSnapshots_Prune(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) - addSnapshot(t, db, "idx", "sha256:a", now.AddDate(0, 0, -100)) - addSnapshot(t, db, "idx", "sha256:b", now.AddDate(0, 0, -60)) - addSnapshot(t, db, "idx", "sha256:c", now.AddDate(0, 0, -10)) - // The only version of another index stays however old it is. - addSnapshot(t, db, "other", "sha256:x", now.AddDate(0, 0, -200)) - - pruned, err := db.PruneMetadataSnapshots("alpine", now.AddDate(0, 0, -30)) - if err != nil { - t.Fatalf("PruneMetadataSnapshots: %v", err) - } - // a was replaced 60 days ago, b only 10 days ago. - if pruned != 1 { - t.Errorf("pruned = %d, want 1", pruned) - } - if got := snapshotDigestAt(t, db, "idx", now.AddDate(0, 0, -90)); got != "" { - t.Errorf("snapshot of a pruned day = %q, want none", got) - } - if got := snapshotDigestAt(t, db, "idx", now.AddDate(0, 0, -50)); got != "sha256:b" { - t.Errorf("snapshot 50 days ago = %q, want sha256:b", got) - } - if got := snapshotDigestAt(t, db, "other", now); got != "sha256:x" { - t.Errorf("only snapshot of other = %q, want sha256:x", got) - } - - due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 10) - if err != nil { - t.Fatal(err) - } - if len(due) != 1 || due[0] != "_snapshots/alpine/idx/sha256:a" { - t.Errorf("queued deletes = %v, want the pruned blob", due) - } - }) -} - -// TestMetadataSnapshots_ConcurrentPostgres records a version while another -// replica's transaction has recorded the same one and not committed yet, as -// two replicas can. The second must wait and then find it, so one row -// remains. Without the lock both inserts succeed. -func TestMetadataSnapshots_ConcurrentPostgres(t *testing.T) { - db := createTestPostgresDB(t) - if db == nil { - t.Skip("PROXY_DATABASE_URL not set, skipping postgres test") - } - defer func() { _ = db.Close() }() - const name = "v3.22/main/x86_64/APKINDEX.tar.gz" - - other, err := db.Beginx() - if err != nil { - t.Fatal(err) - } - defer func() { _ = other.Rollback() }() - if _, err := other.Exec("SELECT pg_advisory_xact_lock($1)", metadataSnapshotLockKey("alpine", name)); err != nil { - t.Fatal(err) - } - if _, err := other.Exec(`INSERT INTO metadata_snapshots (ecosystem, name, storage_path, content_digest, size, valid_from_ms) - VALUES ('alpine', $1, 'other', 'sha256:a', 1, $2)`, name, time.Now().UnixMilli()); err != nil { - t.Fatal(err) - } - - done := make(chan bool, 1) - go func() { done <- addSnapshot(t, db, name, "sha256:a", time.Now()) }() - select { - case added := <-done: - t.Fatalf("recorded without waiting for the other replica (added=%v)", added) - case <-time.After(200 * time.Millisecond): - } - if err := other.Commit(); err != nil { - t.Fatal(err) - } - if added := <-done; added { - t.Error("recorded a version the other replica had just recorded") - } - var n int - if err := db.Get(&n, `SELECT COUNT(*) FROM metadata_snapshots WHERE name = $1`, name); err != nil { - t.Fatal(err) - } - if n != 1 { - t.Errorf("%d snapshot rows, want 1", n) - } - - // Going back to an earlier digest is still recorded. - if !addSnapshot(t, db, name, "sha256:b", time.Now()) || !addSnapshot(t, db, name, "sha256:a", time.Now().Add(time.Millisecond)) { - t.Error("a change back to an earlier digest was not recorded") - } -} diff --git a/internal/database/metadata_stats_test.go b/internal/database/metadata_stats_test.go deleted file mode 100644 index e66e407..0000000 --- a/internal/database/metadata_stats_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package database - -import ( - "database/sql" - "testing" -) - -func TestGetMetadataCacheStats(t *testing.T) { - db := newTestDB(t) - - stats, err := db.GetMetadataCacheStats() - if err != nil { - t.Fatalf("GetMetadataCacheStats on an empty cache: %v", err) - } - if len(stats) != 0 { - t.Fatalf("empty cache returned %+v, want no rows", stats) - } - - rows := []struct { - ecosystem, name string - size sql.NullInt64 - }{ - {"npm", "lodash", sql.NullInt64{Int64: 300, Valid: true}}, - {"npm", "react", sql.NullInt64{Int64: 200, Valid: true}}, - // gem and rubygems are one ecosystem and must share a row. - {"gem", "rails", sql.NullInt64{Int64: 1000, Valid: true}}, - {"rubygems", "rack", sql.NullInt64{Int64: 24, Valid: true}}, - // A row written before sizes were recorded still counts as an entry. - {"pypi", "requests", sql.NullInt64{}}, - } - for _, r := range rows { - if err := db.UpsertMetadataCache(&MetadataCacheEntry{ - Ecosystem: r.ecosystem, - Name: r.name, - StoragePath: "_metadata/" + r.ecosystem + "/" + r.name, - Size: r.size, - }); err != nil { - t.Fatalf("UpsertMetadataCache(%s/%s): %v", r.ecosystem, r.name, err) - } - } - - stats, err = db.GetMetadataCacheStats() - if err != nil { - t.Fatalf("GetMetadataCacheStats: %v", err) - } - - want := []MetadataCacheStats{ - {Ecosystem: "rubygems", Entries: 2, Size: 1024}, - {Ecosystem: "npm", Entries: 2, Size: 500}, - {Ecosystem: "pypi", Entries: 1, Size: 0}, - } - if len(stats) != len(want) { - t.Fatalf("got %+v, want %+v", stats, want) - } - for i := range want { - if stats[i] != want[i] { - t.Errorf("row %d = %+v, want %+v", i, stats[i], want[i]) - } - } -} diff --git a/internal/database/pending_deletes_test.go b/internal/database/pending_deletes_test.go deleted file mode 100644 index f124557..0000000 --- a/internal/database/pending_deletes_test.go +++ /dev/null @@ -1,239 +0,0 @@ -package database - -import ( - "database/sql" - "fmt" - "slices" - "sync" - "testing" - "time" -) - -const ( - pendingVersionPURL = "pkg:npm/pending@1.0.0" - pendingFilename = "pending-1.0.0.tgz" -) - -func upsertPendingArtifact(t *testing.T, db *DB, storagePath string) { - t.Helper() - a := &Artifact{ - VersionPURL: pendingVersionPURL, - Filename: pendingFilename, - UpstreamURL: "https://example.com/" + pendingFilename, - StoragePath: sql.NullString{String: storagePath, Valid: true}, - Size: sql.NullInt64{Int64: 1, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - } - if err := db.UpsertArtifact(a); err != nil { - t.Fatalf("UpsertArtifact(%q): %v", storagePath, err) - } -} - -func recordedPath(t *testing.T, db *DB) sql.NullString { - t.Helper() - a, err := db.GetArtifact(pendingVersionPURL, pendingFilename) - if err != nil || a == nil { - t.Fatalf("GetArtifact: %v, %v", a, err) - } - return a.StoragePath -} - -// allPendingDeletes returns every queued path, due or not. -func allPendingDeletes(t *testing.T, db *DB) []string { - t.Helper() - paths, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 100) - if err != nil { - t.Fatalf("GetDuePendingDeletes: %v", err) - } - return paths -} - -func TestUpsertArtifactQueuesThePathItReplaces(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") - upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") - if got := allPendingDeletes(t, db); len(got) != 0 { - t.Fatalf("queued %v, want nothing while the record keeps its path", got) - } - - upsertPendingArtifact(t, db, "npm/pending/1.0.0/b/pending-1.0.0.tgz") - want := []string{"npm/pending/1.0.0/a/pending-1.0.0.tgz"} - if got := allPendingDeletes(t, db); !slices.Equal(got, want) { - t.Errorf("queued %v, want %v", got, want) - } - }) -} - -// TestUpsertArtifactSkipsRecordThatMoved is the race UpsertArtifact retries: -// another commit moved the record after this one read it, so the write must -// not apply, or the path that commit stored would never be queued. -func TestUpsertArtifactSkipsRecordThatMoved(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") - read := recordedPath(t, db) - upsertPendingArtifact(t, db, "npm/pending/1.0.0/b/pending-1.0.0.tgz") - - late := &Artifact{ - VersionPURL: pendingVersionPURL, - Filename: pendingFilename, - UpstreamURL: "https://example.com/" + pendingFilename, - StoragePath: sql.NullString{String: "npm/pending/1.0.0/c/pending-1.0.0.tgz", Valid: true}, - } - applied, err := db.upsertArtifactFrom(db, late, read) - if err != nil { - t.Fatalf("upsertArtifactFrom: %v", err) - } - if applied { - t.Error("write applied over a record that moved since it was read") - } - if got := recordedPath(t, db).String; got != "npm/pending/1.0.0/b/pending-1.0.0.tgz" { - t.Errorf("record points at %q, want the newer commit kept", got) - } - - missing := &Artifact{VersionPURL: "pkg:npm/pending@2.0.0", Filename: pendingFilename, UpstreamURL: "u"} - applied, err = db.upsertArtifactFrom(db, missing, sql.NullString{}) - if err != nil || !applied { - t.Errorf("insert of a new record: applied=%v err=%v", applied, err) - } - }) -} - -// TestConcurrentUpsertsQueueEveryReplacedPath commits one artifact from several -// fetches at once: whichever commit ends up recorded, every other path must be -// queued, or its object is never deleted. -func TestConcurrentUpsertsQueueEveryReplacedPath(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - // A commit only retries after another commit succeeded, so this many - // always fit within upsertArtifactAttempts. - const commits = upsertArtifactAttempts - 1 - paths := make([]string, commits) - errs := make(chan error, commits) - var wg sync.WaitGroup - for i := range paths { - paths[i] = fmt.Sprintf("npm/pending/1.0.0/%d/pending-1.0.0.tgz", i) - wg.Go(func() { - errs <- db.UpsertArtifact(&Artifact{ - VersionPURL: pendingVersionPURL, - Filename: pendingFilename, - UpstreamURL: "https://example.com/" + pendingFilename, - StoragePath: sql.NullString{String: paths[i], Valid: true}, - }) - }) - } - wg.Wait() - close(errs) - for err := range errs { - if err != nil { - t.Fatalf("UpsertArtifact: %v", err) - } - } - - recorded := recordedPath(t, db).String - want := slices.DeleteFunc(slices.Clone(paths), func(p string) bool { return p == recorded }) - got := allPendingDeletes(t, db) - slices.Sort(want) - slices.Sort(got) - if !slices.Equal(got, want) { - t.Errorf("recorded %q, queued %v, want %v", recorded, got, want) - } - }) -} - -func TestClearArtifactCacheLeavesRecordThatMoved(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") - upsertPendingArtifact(t, db, "npm/pending/1.0.0/b/pending-1.0.0.tgz") - - if cleared, err := db.ClearArtifactCache(pendingVersionPURL, pendingFilename, "npm/pending/1.0.0/a/pending-1.0.0.tgz"); err != nil || cleared { - t.Fatalf("ClearArtifactCache: cleared=%v err=%v, want nothing cleared", cleared, err) - } - if err := db.DiscardArtifact(pendingVersionPURL, pendingFilename, "npm/pending/1.0.0/a/pending-1.0.0.tgz"); err != nil { - t.Fatalf("DiscardArtifact: %v", err) - } - if got := recordedPath(t, db).String; got != "npm/pending/1.0.0/b/pending-1.0.0.tgz" { - t.Errorf("record points at %q, want the newer commit kept", got) - } - }) -} - -func TestDiscardArtifactClearsAndQueues(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") - - if err := db.DiscardArtifact(pendingVersionPURL, pendingFilename, "npm/pending/1.0.0/a/pending-1.0.0.tgz"); err != nil { - t.Fatalf("DiscardArtifact: %v", err) - } - if got := recordedPath(t, db); got.Valid { - t.Errorf("record still points at %q", got.String) - } - want := []string{"npm/pending/1.0.0/a/pending-1.0.0.tgz"} - if got := allPendingDeletes(t, db); !slices.Equal(got, want) { - t.Errorf("queued %v, want %v", got, want) - } - }) -} - -// When the replaced path can't be queued, the write is rolled back, so the -// record keeps pointing at the old object instead of orphaning it. -func TestUpsertArtifactRollsBackWhenQueueingFails(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") - if _, err := db.Exec(`DROP TABLE pending_deletes`); err != nil { - t.Fatalf("dropping pending_deletes: %v", err) - } - - err := db.UpsertArtifact(&Artifact{ - VersionPURL: pendingVersionPURL, - Filename: pendingFilename, - UpstreamURL: "https://example.com/" + pendingFilename, - StoragePath: sql.NullString{String: "npm/pending/1.0.0/b/pending-1.0.0.tgz", Valid: true}, - }) - if err == nil { - t.Fatal("UpsertArtifact succeeded without queueing the replaced path") - } - if got := recordedPath(t, db).String; got != "npm/pending/1.0.0/a/pending-1.0.0.tgz" { - t.Errorf("record points at %q, want the old path kept", got) - } - }) -} - -func TestDiscardArtifactRollsBackWhenQueueingFails(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") - if _, err := db.Exec(`DROP TABLE pending_deletes`); err != nil { - t.Fatalf("dropping pending_deletes: %v", err) - } - - if err := db.DiscardArtifact(pendingVersionPURL, pendingFilename, "npm/pending/1.0.0/a/pending-1.0.0.tgz"); err == nil { - t.Fatal("DiscardArtifact succeeded without queueing the path") - } - if got := recordedPath(t, db).String; got != "npm/pending/1.0.0/a/pending-1.0.0.tgz" { - t.Errorf("record points at %q, want it kept", got) - } - }) -} - -func TestGetDuePendingDeletes(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - for _, path := range []string{"old", "referenced"} { - if err := db.QueuePendingDelete(path); err != nil { - t.Fatalf("QueuePendingDelete: %v", err) - } - } - upsertPendingArtifact(t, db, "referenced") - - if got, err := db.GetDuePendingDeletes(time.Now().Add(-time.Hour), 100); err != nil || len(got) != 0 { - t.Errorf("before the grace period: got %v (err %v), want nothing", got, err) - } - if got := allPendingDeletes(t, db); !slices.Equal(got, []string{"old"}) { - t.Errorf("after the grace period: got %v, want [old] and not the path a record points at", got) - } - - if err := db.RemovePendingDelete("old"); err != nil { - t.Fatalf("RemovePendingDelete: %v", err) - } - if got := allPendingDeletes(t, db); len(got) != 0 { - t.Errorf("after removal: got %v", got) - } - }) -} diff --git a/internal/database/postgres_pool_test.go b/internal/database/postgres_pool_test.go deleted file mode 100644 index bd39bee..0000000 --- a/internal/database/postgres_pool_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package database - -import ( - "context" - "database/sql" - "os" - "testing" -) - -// TestOpenPostgresKeepsConnectionsIdle checks the connection-count limits -// OpenPostgres sets: the open cap admits a burst of postgresMaxIdleConns -// connections, and releasing them again leaves all of them idle in the pool. -// database/sql's default keeps only two, so the next burst would open a new -// Postgres session for almost every request. The idle-time and lifetime -// settings are not exercised here. -func TestOpenPostgresKeepsConnectionsIdle(t *testing.T) { - url := os.Getenv("PROXY_DATABASE_URL") - if url == "" { - t.Skip("PROXY_DATABASE_URL not set, skipping postgres pool test") - } - - db, err := OpenPostgres(url) - if err != nil { - t.Fatalf("OpenPostgres failed: %v", err) - } - defer func() { _ = db.Close() }() - - const burst = postgresMaxIdleConns - // database/sql keeps two idle connections by default; a burst that small - // could not tell the tuned pool from the default one. - if burst <= 2 { - t.Fatalf("postgresMaxIdleConns = %d, want more than database/sql's default of 2", burst) - } - if got := db.Stats().MaxOpenConnections; got <= 0 || got < burst { - t.Fatalf("MaxOpenConnections = %d, want a cap of at least %d", got, burst) - } - - conns := make([]*sql.Conn, 0, burst) - for range burst { - conn, err := db.Conn(context.Background()) - if err != nil { - t.Fatalf("taking connection %d: %v", len(conns)+1, err) - } - t.Cleanup(func() { _ = conn.Close() }) // release the session if an assertion below fails - conns = append(conns, conn) - } - if got := db.Stats().InUse; got != burst { - t.Fatalf("InUse = %d while holding %d connections", got, burst) - } - - for _, conn := range conns { - _ = conn.Close() - } - if got := db.Stats().Idle; got != burst { - t.Errorf("Idle = %d after releasing %d connections, want all of them kept", got, burst) - } -} diff --git a/internal/database/projects.go b/internal/database/projects.go deleted file mode 100644 index c14b0d6..0000000 --- a/internal/database/projects.go +++ /dev/null @@ -1,222 +0,0 @@ -package database - -import ( - "database/sql" - "errors" - "fmt" - "time" -) - -// Project is a repository that uploaded an SBOM to the project inventory. -type Project struct { - ID int64 `db:"id"` - Name string `db:"name"` - Ref sql.NullString `db:"ref"` - Branch sql.NullString `db:"branch"` - UploadedAt time.Time `db:"uploaded_at"` - // Components is the number of package versions recorded for it. - Components int64 `db:"components"` -} - -// ProjectComponent is a package version a project uses. PackagePURL is the -// unversioned package URL, as packages.purl records it. -type ProjectComponent struct { - PackagePURL string `db:"package_purl"` - Ecosystem string `db:"ecosystem"` - Name string `db:"name"` - Version string `db:"version"` -} - -// ProjectComponentInfo is a project's component together with the package -// record the proxy holds for it, if any. -type ProjectComponentInfo struct { - ProjectComponent - CachedEcosystem sql.NullString `db:"cached_ecosystem"` - CachedName sql.NullString `db:"cached_name"` - LatestVersion sql.NullString `db:"latest_version"` -} - -// ProjectUsage is a project using a version of a package. -type ProjectUsage struct { - Name string `db:"name"` - Ref sql.NullString `db:"ref"` - Branch sql.NullString `db:"branch"` - UploadedAt time.Time `db:"uploaded_at"` - Version string `db:"version"` -} - -const projectColumns = ` - SELECT pr.id, pr.name, pr.ref, pr.branch, pr.uploaded_at, - (SELECT COUNT(*) FROM project_components pc WHERE pc.project_id = pr.id) AS components` - -// SaveProject records the components of a project, replacing everything -// recorded for it before, in one transaction. Readers see either the old or -// the new set, never a mix. -func (db *DB) SaveProject(name, ref, branch string, uploadedAt time.Time, components []ProjectComponent) error { - tx, err := db.Beginx() - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - // The upsert locks the project row, so concurrent uploads of the same - // project and pruning take turns. - if _, err := tx.Exec(db.Rebind(` - INSERT INTO projects (name, ref, branch, uploaded_at) VALUES (?, ?, ?, ?) - ON CONFLICT(name) DO UPDATE SET ref = excluded.ref, branch = excluded.branch, uploaded_at = excluded.uploaded_at - `), name, nullString(ref), nullString(branch), uploadedAt.UTC()); err != nil { - return fmt.Errorf("recording project: %w", err) - } - var id int64 - if err := tx.Get(&id, db.Rebind(`SELECT id FROM projects WHERE name = ?`), name); err != nil { - return fmt.Errorf("reading project id: %w", err) - } - if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { - return fmt.Errorf("removing old components: %w", err) - } - - insert, err := tx.Preparex(db.Rebind(` - INSERT INTO project_components (project_id, package_purl, ecosystem, name, version) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(project_id, package_purl, version) DO NOTHING - `)) - if err != nil { - return err - } - defer func() { _ = insert.Close() }() - for _, c := range components { - if _, err := insert.Exec(id, c.PackagePURL, c.Ecosystem, c.Name, c.Version); err != nil { - return fmt.Errorf("recording component %s@%s: %w", c.PackagePURL, c.Version, err) - } - } - return tx.Commit() -} - -// DeleteProject removes a project and its components. It reports whether the -// project existed. -func (db *DB) DeleteProject(name string) (bool, error) { - tx, err := db.Beginx() - if err != nil { - return false, err - } - defer func() { _ = tx.Rollback() }() - - var id int64 - err = tx.Get(&id, db.Rebind(`DELETE FROM projects WHERE name = ? RETURNING id`), name) - if errors.Is(err, sql.ErrNoRows) { - return false, nil - } - if err != nil { - return false, fmt.Errorf("deleting project: %w", err) - } - if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { - return false, fmt.Errorf("deleting project components: %w", err) - } - return true, tx.Commit() -} - -// PruneProjects removes the projects last uploaded before cutoff, with their -// components, and returns how many it removed. A project uploaded again while -// the prune runs is kept. -func (db *DB) PruneProjects(cutoff time.Time) (int, error) { - var ids []int64 - if err := db.Select(&ids, db.Rebind(`SELECT id FROM projects WHERE uploaded_at < ?`), cutoff.UTC()); err != nil { - return 0, fmt.Errorf("listing expired projects: %w", err) - } - pruned := 0 - for _, id := range ids { - ok, err := db.pruneProject(id, cutoff) - if err != nil { - return pruned, err - } - if ok { - pruned++ - } - } - return pruned, nil -} - -func (db *DB) pruneProject(id int64, cutoff time.Time) (bool, error) { - tx, err := db.Beginx() - if err != nil { - return false, err - } - defer func() { _ = tx.Rollback() }() - - res, err := tx.Exec(db.Rebind(`DELETE FROM projects WHERE id = ? AND uploaded_at < ?`), id, cutoff.UTC()) - if err != nil { - return false, fmt.Errorf("pruning project: %w", err) - } - if n, err := res.RowsAffected(); err != nil || n == 0 { - return false, err - } - if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { - return false, fmt.Errorf("pruning project components: %w", err) - } - return true, tx.Commit() -} - -// ListProjects returns all projects ordered by name. -func (db *DB) ListProjects() ([]Project, error) { - projects := []Project{} - if err := db.Select(&projects, projectColumns+` FROM projects pr ORDER BY pr.name`); err != nil { - return nil, err - } - return projects, nil -} - -// GetProject returns the named project, or nil if there is none. -func (db *DB) GetProject(name string) (*Project, error) { - var p Project - err := db.Get(&p, db.Rebind(projectColumns+` FROM projects pr WHERE pr.name = ?`), name) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, err - } - return &p, nil -} - -// GetProjectComponents returns the components of a project, ordered by -// ecosystem, name and version, with the package record the proxy holds for -// each. -func (db *DB) GetProjectComponents(projectID int64) ([]ProjectComponentInfo, error) { - components := []ProjectComponentInfo{} - err := db.Select(&components, db.Rebind(` - SELECT c.package_purl, c.ecosystem, c.name, c.version, - p.ecosystem AS cached_ecosystem, p.name AS cached_name, p.latest_version - FROM project_components c - LEFT JOIN packages p ON p.purl = c.package_purl - WHERE c.project_id = ? - ORDER BY c.ecosystem, c.name, c.version - `), projectID) - if err != nil { - return nil, err - } - return components, nil -} - -// GetProjectsUsingPackage returns the projects using a package, or only -// those using version when it is not empty, ordered by project and version. -func (db *DB) GetProjectsUsingPackage(packagePURL, version string) ([]ProjectUsage, error) { - query := ` - SELECT pr.name, pr.ref, pr.branch, pr.uploaded_at, c.version - FROM project_components c - JOIN projects pr ON pr.id = c.project_id - WHERE c.package_purl = ?` - args := []any{packagePURL} - if version != "" { - query += ` AND c.version = ?` - args = append(args, version) - } - usages := []ProjectUsage{} - if err := db.Select(&usages, db.Rebind(query+` ORDER BY pr.name, c.version`), args...); err != nil { - return nil, err - } - return usages, nil -} - -func nullString(s string) sql.NullString { - return sql.NullString{String: s, Valid: s != ""} -} diff --git a/internal/database/projects_test.go b/internal/database/projects_test.go deleted file mode 100644 index 27bb34c..0000000 --- a/internal/database/projects_test.go +++ /dev/null @@ -1,259 +0,0 @@ -package database - -import ( - "database/sql" - "slices" - "sync" - "testing" - "time" -) - -var ( - lodash20 = ProjectComponent{PackagePURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", Version: "4.17.20"} - lodash21 = ProjectComponent{PackagePURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", Version: "4.17.21"} - console = ProjectComponent{PackagePURL: "pkg:composer/symfony/console", Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1"} -) - -func saveProject(t *testing.T, db *DB, name string, at time.Time, components ...ProjectComponent) { - t.Helper() - if err := db.SaveProject(name, "", "", at, components); err != nil { - t.Fatalf("SaveProject(%s): %v", name, err) - } -} - -func usageNames(usages []ProjectUsage) []string { - var names []string - for _, u := range usages { - names = append(names, u.Name+"@"+u.Version) - } - return names -} - -func TestSaveProjectReplacesComponents(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now().UTC().Truncate(time.Second) - if err := db.SaveProject("acme/web", "abc123", "main", now, []ProjectComponent{lodash20, console, lodash20}); err != nil { - t.Fatal(err) - } - saveProject(t, db, "acme/web", now.Add(time.Minute), lodash21) - - project, err := db.GetProject("acme/web") - if err != nil || project == nil { - t.Fatalf("GetProject = %v, %v", project, err) - } - if project.Components != 1 || project.Ref.Valid || project.Branch.Valid { - t.Errorf("project = %+v, want one component and ref and branch cleared", project) - } - if !project.UploadedAt.Equal(now.Add(time.Minute)) { - t.Errorf("uploaded_at = %v, want %v", project.UploadedAt, now.Add(time.Minute)) - } - components, err := db.GetProjectComponents(project.ID) - if err != nil { - t.Fatal(err) - } - if len(components) != 1 || components[0].ProjectComponent != lodash21 { - t.Errorf("components = %+v, want only lodash 4.17.21", components) - } - - missing, err := db.GetProject("acme/other") - if err != nil || missing != nil { - t.Errorf("GetProject(missing) = %v, %v", missing, err) - } - }) -} - -func TestGetProjectsUsingPackage(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now() - saveProject(t, db, "acme/web", now, lodash20, console) - saveProject(t, db, "acme/api", now, lodash21) - saveProject(t, db, "acme/cli", now, console) - - all, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "") - if err != nil { - t.Fatal(err) - } - if got, want := usageNames(all), []string{"acme/api@4.17.21", "acme/web@4.17.20"}; !slices.Equal(got, want) { - t.Errorf("any version = %v, want %v", got, want) - } - one, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "4.17.20") - if err != nil { - t.Fatal(err) - } - if got, want := usageNames(one), []string{"acme/web@4.17.20"}; !slices.Equal(got, want) { - t.Errorf("4.17.20 = %v, want %v", got, want) - } - none, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "1.0.0") - if err != nil || len(none) != 0 { - t.Errorf("unused version = %v, %v", none, err) - } - }) -} - -func TestListAndDeleteProjects(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now() - saveProject(t, db, "acme/web", now, lodash20, console) - saveProject(t, db, "acme/api", now) - - projects, err := db.ListProjects() - if err != nil { - t.Fatal(err) - } - if len(projects) != 2 || projects[0].Name != "acme/api" || projects[0].Components != 0 || projects[1].Components != 2 { - t.Fatalf("projects = %+v", projects) - } - - deleted, err := db.DeleteProject("acme/web") - if err != nil || !deleted { - t.Fatalf("DeleteProject = %v, %v", deleted, err) - } - deleted, err = db.DeleteProject("acme/web") - if err != nil || deleted { - t.Errorf("second DeleteProject = %v, %v, want false", deleted, err) - } - var rows int - if err := db.Get(&rows, `SELECT COUNT(*) FROM project_components`); err != nil { - t.Fatal(err) - } - if rows != 0 { - t.Errorf("%d component rows left after the delete", rows) - } - }) -} - -func TestPruneProjects(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now() - saveProject(t, db, "acme/old", now.Add(-100*24*time.Hour), lodash20) - saveProject(t, db, "acme/new", now.Add(-time.Hour), lodash21) - - pruned, err := db.PruneProjects(now.Add(-90 * 24 * time.Hour)) - if err != nil { - t.Fatal(err) - } - if pruned != 1 { - t.Errorf("pruned = %d, want 1", pruned) - } - projects, err := db.ListProjects() - if err != nil { - t.Fatal(err) - } - if len(projects) != 1 || projects[0].Name != "acme/new" { - t.Errorf("projects after prune = %+v", projects) - } - usages, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "") - if err != nil { - t.Fatal(err) - } - if got, want := usageNames(usages), []string{"acme/new@4.17.21"}; !slices.Equal(got, want) { - t.Errorf("usages after prune = %v, want %v", got, want) - } - }) -} - -func TestProjectComponentsJoinPackages(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - if err := db.UpsertPackage(&Package{ - PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", - LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, - }); err != nil { - t.Fatal(err) - } - saveProject(t, db, "acme/web", time.Now(), lodash20, console) - saveProject(t, db, "acme/api", time.Now(), lodash21) - - project, err := db.GetProject("acme/web") - if err != nil || project == nil { - t.Fatalf("GetProject = %v, %v", project, err) - } - components, err := db.GetProjectComponents(project.ID) - if err != nil { - t.Fatal(err) - } - if len(components) != 2 { - t.Fatalf("components = %+v", components) - } - // Ordered by ecosystem: composer first, which the proxy has no record of. - if components[0].CachedName.Valid { - t.Errorf("uncached component has a package record: %+v", components[0]) - } - if components[1].CachedName.String != "lodash" || components[1].LatestVersion.String != "4.17.21" { - t.Errorf("cached component = %+v", components[1]) - } - - }) -} - -func TestMigrateAddsProjects(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - // A database from before the project inventory. - for _, table := range []string{"project_components", "projects"} { - if _, err := db.Exec("DROP TABLE " + table); err != nil { - t.Fatalf("dropping %s: %v", table, err) - } - } - if _, err := db.Exec(db.Rebind("DELETE FROM migrations WHERE name = ?"), "021_add_projects"); err != nil { - t.Fatalf("forgetting the migration: %v", err) - } - if err := db.MigrateSchema(); err != nil { - t.Fatalf("MigrateSchema: %v", err) - } - saveProject(t, db, "acme/web", time.Now(), lodash20) - usages, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "4.17.20") - if err != nil || len(usages) != 1 { - t.Fatalf("GetProjectsUsingPackage after migration = %v, %v", usages, err) - } - }) -} - -// TestSaveProjectConcurrentUploads checks that concurrent uploads of one -// project all succeed and leave the components of exactly one of them. -func TestSaveProjectConcurrentUploads(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - const uploads = 8 - sets := make([][]ProjectComponent, uploads) - for i := range sets { - for j := 0; j <= i; j++ { - sets[i] = append(sets[i], ProjectComponent{ - PackagePURL: "pkg:npm/dep" + string(rune('a'+j)), Ecosystem: "npm", - Name: "dep" + string(rune('a'+j)), Version: "1.0." + string(rune('0'+i)), - }) - } - } - now := time.Now().UTC() - errs := make(chan error, uploads) - var wg sync.WaitGroup - for i := range uploads { - wg.Go(func() { errs <- db.SaveProject("acme/web", "", "", now, sets[i]) }) - } - wg.Wait() - close(errs) - for err := range errs { - if err != nil { - t.Errorf("SaveProject: %v", err) - } - } - - project, err := db.GetProject("acme/web") - if err != nil || project == nil { - t.Fatalf("GetProject = %v, %v", project, err) - } - components, err := db.GetProjectComponents(project.ID) - if err != nil { - t.Fatal(err) - } - if len(components) == 0 || len(components) > uploads { - t.Fatalf("%d components, want those of one upload", len(components)) - } - version := components[0].Version - for _, c := range components { - if c.Version != version { - t.Fatalf("components mix uploads: %+v", components) - } - } - if want := int(version[len(version)-1]-'0') + 1; len(components) != want { - t.Errorf("%d components of the upload with version %s, want %d", len(components), version, want) - } - }) -} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7eaa6e..5d95596 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -2,16 +2,8 @@ package database import ( "database/sql" - "errors" "fmt" - "slices" - "sort" - "strings" "time" - - "github.com/git-pkgs/artifacts" - "github.com/git-pkgs/purl" - "github.com/opencontainers/go-digest" ) // Package queries @@ -54,17 +46,7 @@ func (db *DB) GetPackageByEcosystemName(ecosystem, name string) (*Package, error return &pkg, nil } -// execer runs statements on the database or inside a transaction. -type execer interface { - Exec(query string, args ...any) (sql.Result, error) - Get(dest any, query string, args ...any) error -} - func (db *DB) UpsertPackage(pkg *Package) error { - return db.upsertPackage(db, pkg) -} - -func (db *DB) upsertPackage(exec execer, pkg *Package) error { now := time.Now() var query string @@ -102,7 +84,7 @@ func (db *DB) upsertPackage(exec execer, pkg *Package) error { ` } - _, err := exec.Exec(query, + _, err := db.Exec(query, pkg.PURL, pkg.Ecosystem, pkg.Name, pkg.LatestVersion, pkg.License, pkg.Description, pkg.Homepage, pkg.RepositoryURL, pkg.RegistryURL, pkg.EnrichedAt, now, now, @@ -148,10 +130,6 @@ func (db *DB) GetVersionsByPackagePURL(packagePURL string) ([]Version, error) { } func (db *DB) UpsertVersion(v *Version) error { - return db.upsertVersion(db, v) -} - -func (db *DB) upsertVersion(exec execer, v *Version) error { now := time.Now() var query string @@ -163,7 +141,7 @@ func (db *DB) upsertVersion(exec execer, v *Version) error { ON CONFLICT(purl) DO UPDATE SET license = EXCLUDED.license, integrity = EXCLUDED.integrity, - published_at = COALESCE(EXCLUDED.published_at, versions.published_at), + published_at = EXCLUDED.published_at, yanked = EXCLUDED.yanked, enriched_at = EXCLUDED.enriched_at, updated_at = EXCLUDED.updated_at @@ -176,16 +154,16 @@ func (db *DB) upsertVersion(exec execer, v *Version) error { ON CONFLICT(purl) DO UPDATE SET license = excluded.license, integrity = excluded.integrity, - published_at = COALESCE(excluded.published_at, published_at), + published_at = excluded.published_at, yanked = excluded.yanked, enriched_at = excluded.enriched_at, updated_at = excluded.updated_at ` } - _, err := exec.Exec(query, + _, err := db.Exec(query, v.PURL, v.PackagePURL, v.License, v.Integrity, - inUTC(v.PublishedAt), v.Yanked, v.EnrichedAt, now, now, + v.PublishedAt, v.Yanked, v.EnrichedAt, now, now, ) if err != nil { return fmt.Errorf("upserting version: %w", err) @@ -193,57 +171,6 @@ func (db *DB) upsertVersion(exec execer, v *Version) error { return nil } -// SetVersionPublishedAt records a version's publish time, creating the -// versions row if the proxy has not seen the version yet. It only writes -// published_at, so it never disturbs enrichment data on an existing row. -func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { - now := time.Now() - var query string - - if db.dialect == DialectPostgres { - query = ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT(purl) DO UPDATE SET - published_at = EXCLUDED.published_at, - updated_at = EXCLUDED.updated_at - ` - } else { - query = ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(purl) DO UPDATE SET - published_at = excluded.published_at, - updated_at = excluded.updated_at - ` - } - - _, err := db.Exec(query, versionPURL, packagePURL, publishedAt.UTC(), now, now) - if err != nil { - return fmt.Errorf("setting version publish time: %w", err) - } - return nil -} - -// GetFetchedVersionPURLs returns the versions of a package that have an -// artifact row, cached now or evicted since. Versions without one were only -// seen in metadata. -func (db *DB) GetFetchedVersionPURLs(packagePURL string) (map[string]bool, error) { - var purls []string - query := db.Rebind(` - SELECT DISTINCT version_purl FROM artifacts - WHERE version_purl IN (SELECT purl FROM versions WHERE package_purl = ?) - `) - if err := db.Select(&purls, query, packagePURL); err != nil { - return nil, fmt.Errorf("listing fetched versions: %w", err) - } - fetched := make(map[string]bool, len(purls)) - for _, p := range purls { - fetched[p] = true - } - return fetched, nil -} - // Artifact queries func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { @@ -251,7 +178,7 @@ func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at, source + created_at, updated_at FROM artifacts WHERE version_purl = ? AND filename = ? `) err := db.Get(&a, query, versionPURL, filename) @@ -264,90 +191,12 @@ func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { return &a, nil } -// GetCachedArtifact returns the fields needed to serve a cached artifact. -func (db *DB) GetCachedArtifact(packagePURL, versionPURL, filename string) (*CachedArtifact, error) { - var row cachedArtifactRow - query := db.Rebind(` - SELECT packages.ecosystem, artifacts.storage_path, artifacts.content_hash, artifacts.size, - artifacts.content_type, versions.integrity - FROM artifacts - JOIN versions ON versions.purl = artifacts.version_purl - JOIN packages ON packages.purl = versions.package_purl - WHERE packages.purl = ? AND artifacts.version_purl = ? AND artifacts.filename = ? - AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL - `) - err := db.Get(&row, query, packagePURL, versionPURL, filename) - if err == sql.ErrNoRows { - return nil, nil - } - if err != nil { - return nil, err - } - return row.artifact(versionPURL, filename), nil -} - -// StoredContent is a stored object with a given content hash. -type StoredContent struct { - StoragePath string `db:"storage_path"` - Size int64 `db:"size"` - ContentType sql.NullString `db:"content_type"` -} - -// FindStoredContent returns a cached artifact whose content has the sha256 -// hash, under any name, or nil if none is stored. Pushed artifacts are left -// out. -func (db *DB) FindStoredContent(hash string) (*StoredContent, error) { - var found StoredContent - query := db.Rebind(` - SELECT storage_path, size, content_type FROM artifacts - WHERE content_hash = ? AND storage_path IS NOT NULL AND fetched_at IS NOT NULL - AND size IS NOT NULL AND source <> 'local' - LIMIT 1 - `) - err := db.Get(&found, query, hash) - if err == sql.ErrNoRows { - return nil, nil - } - if err != nil { - return nil, err - } - return &found, nil -} - -type cachedArtifactRow struct { - Ecosystem string `db:"ecosystem"` - StoragePath string `db:"storage_path"` - ContentHash sql.NullString `db:"content_hash"` - Size sql.NullInt64 `db:"size"` - ContentType sql.NullString `db:"content_type"` - Integrity sql.NullString `db:"integrity"` -} - -// artifact converts a cached artifact row to a CachedArtifact without -// validation. A malformed hash or integrity value is handled by -// checkCache, which clears the record and treats the request as a cache -// miss so the client is served a fresh fetch instead of an error. -func (row cachedArtifactRow) artifact(versionPURL, filename string) *CachedArtifact { - return &CachedArtifact{ - Ecosystem: row.Ecosystem, - StoragePath: row.StoragePath, - Integrity: row.Integrity, - Artifact: artifacts.Artifact{ - PURL: versionPURL, - Digest: digest.Digest("sha256:" + row.ContentHash.String), - Size: row.Size.Int64, - Filename: filename, - MediaType: row.ContentType.String, - }, - } -} - func (db *DB) GetArtifactByPath(storagePath string) (*Artifact, error) { var a Artifact query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at, source + created_at, updated_at FROM artifacts WHERE storage_path = ? `) err := db.Get(&a, query, storagePath) @@ -365,7 +214,7 @@ func (db *DB) GetArtifactsByVersionPURL(versionPURL string) ([]Artifact, error) query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at, source + created_at, updated_at FROM artifacts WHERE version_purl = ? ORDER BY filename `) @@ -376,108 +225,7 @@ func (db *DB) GetArtifactsByVersionPURL(versionPURL string) ([]Artifact, error) return artifacts, nil } -// upsertArtifactAttempts bounds how often UpsertArtifact retries a record that -// concurrent commits keep moving. -const upsertArtifactAttempts = 5 - -// UpsertArtifact records a. A storage path the record stops pointing at is -// queued for deletion, since each fetch stores its own object and nothing else -// would remove it. The write applies only if the record still holds the path -// read before it, so commits racing on one artifact each queue the path they -// replaced. -// inUTC returns t in UTC. Artifacts store fetched_at and last_accessed_at in -// UTC: lib/pq sends a time with its offset, a Postgres TIMESTAMP column keeps -// only the wall clock, and lib/pq reads that back as UTC, so anything else -// comes back shifted by the local offset. -func inUTC(t sql.NullTime) sql.NullTime { - if t.Valid { - t.Time = t.Time.UTC() - } - return t -} - func (db *DB) UpsertArtifact(a *Artifact) error { - return db.recordArtifact(a, nil, ErrLocalArtifactExists) -} - -// ErrLocalArtifactExists is returned when an artifact would replace one that -// was pushed to the proxy. Pushed artifacts are only removed by deleting them. -var ErrLocalArtifactExists = errors.New("a pushed artifact already exists") - -// RecordCachedArtifact records an artifact fetched from an upstream together -// with its package and version, in one transaction. If the artifact was -// pushed in the meantime, nothing is written and ErrLocalArtifactExists is -// returned, so the push's version integrity is kept as well. -func (db *DB) RecordCachedArtifact(pkg *Package, v *Version, a *Artifact) error { - return db.recordArtifact(a, func(tx execer) error { - if err := db.upsertPackage(tx, pkg); err != nil { - return err - } - return db.upsertVersion(tx, v) - }, ErrLocalArtifactExists) -} - -// artifactSlot is what an artifact record holds before it is replaced. -type artifactSlot struct { - StoragePath sql.NullString `db:"storage_path"` - Source sql.NullString `db:"source"` -} - -// recordArtifact writes before's records and a in one transaction. A pushed -// artifact is never replaced: the transaction is rolled back and localErr -// returned. A storage path the record stops pointing at is queued for -// deletion in the same transaction, since each fetch or push stores its own -// object and nothing else would remove it. The write applies only if the -// record still holds the path read before it, so commits racing on one -// artifact each queue the path they replaced. -func (db *DB) recordArtifact(a *Artifact, before func(tx execer) error, localErr error) error { - for range upsertArtifactAttempts { - done, err := db.tryRecordArtifact(a, before, localErr) - if err != nil || done { - return err - } - } - return errors.New("recording artifact: record kept changing") -} - -func (db *DB) tryRecordArtifact(a *Artifact, before func(tx execer) error, localErr error) (bool, error) { - tx, err := db.Beginx() - if err != nil { - return false, err - } - defer func() { _ = tx.Rollback() }() - - if before != nil { - if err := before(tx); err != nil { - return false, err - } - } - - var current artifactSlot - query := db.Rebind(`SELECT storage_path, source FROM artifacts WHERE version_purl = ? AND filename = ?`) - if err := tx.Get(¤t, query, a.VersionPURL, a.Filename); err != nil && !errors.Is(err, sql.ErrNoRows) { - return false, fmt.Errorf("reading artifact: %w", err) - } - if current.Source.String == SourceLocal { - return false, localErr - } - - applied, err := db.upsertArtifactFrom(tx, a, current.StoragePath) - if err != nil || !applied { - return false, err - } - previous := current.StoragePath - if previous.Valid && previous.String != a.StoragePath.String { - if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), previous.String, time.Now().UTC()); err != nil { - return false, fmt.Errorf("queueing replaced artifact: %w", err) - } - } - return true, tx.Commit() -} - -// upsertArtifactFrom writes a if the record is absent, or still holds -// previous and wasn't pushed, and reports whether it did. -func (db *DB) upsertArtifactFrom(exec execer, a *Artifact, previous sql.NullString) (bool, error) { now := time.Now() var query string @@ -485,69 +233,50 @@ func (db *DB) upsertArtifactFrom(exec execer, a *Artifact, previous sql.NullStri query = ` INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at, source, manifest) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT(version_purl, filename) DO UPDATE SET - upstream_url = EXCLUDED.upstream_url, storage_path = EXCLUDED.storage_path, content_hash = EXCLUDED.content_hash, size = EXCLUDED.size, content_type = EXCLUDED.content_type, fetched_at = EXCLUDED.fetched_at, - updated_at = EXCLUDED.updated_at, - source = EXCLUDED.source, - manifest = EXCLUDED.manifest - WHERE artifacts.storage_path IS NOT DISTINCT FROM $15 - AND (artifacts.source IS NULL OR artifacts.source <> 'local') + updated_at = EXCLUDED.updated_at ` } else { query = ` INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at, source, manifest) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(version_purl, filename) DO UPDATE SET - upstream_url = excluded.upstream_url, storage_path = excluded.storage_path, content_hash = excluded.content_hash, size = excluded.size, content_type = excluded.content_type, fetched_at = excluded.fetched_at, - updated_at = excluded.updated_at, - source = excluded.source, - manifest = excluded.manifest - WHERE artifacts.storage_path IS ? - AND (artifacts.source IS NULL OR artifacts.source <> 'local') + updated_at = excluded.updated_at ` } - res, err := exec.Exec(query, + _, err := db.Exec(query, a.VersionPURL, a.Filename, a.UpstreamURL, a.StoragePath, a.ContentHash, - a.Size, a.ContentType, inUTC(a.FetchedAt), a.HitCount, inUTC(a.LastAccessedAt), now, now, - a.sourceOrUpstream(), a.Manifest, previous, + a.Size, a.ContentType, a.FetchedAt, a.HitCount, a.LastAccessedAt, now, now, ) if err != nil { - return false, fmt.Errorf("upserting artifact: %w", err) + return fmt.Errorf("upserting artifact: %w", err) } - n, err := res.RowsAffected() - if err != nil { - return false, fmt.Errorf("upserting artifact: %w", err) - } - return n > 0, nil + return nil } func (db *DB) RecordArtifactHit(versionPURL, filename string) error { now := time.Now() - if db.hits != nil { - db.hits.add(hitKey{versionPURL, filename}, hitEntry{count: 1, last: now}) - return nil - } query := db.Rebind(` UPDATE artifacts SET hit_count = hit_count + 1, last_accessed_at = ?, updated_at = ? WHERE version_purl = ? AND filename = ? `) - _, err := db.Exec(query, now.UTC(), now, versionPURL, filename) + _, err := db.Exec(query, now, now, versionPURL, filename) return err } @@ -559,7 +288,7 @@ func (db *DB) MarkArtifactCached(versionPURL, filename, storagePath, contentHash fetched_at = ?, updated_at = ? WHERE version_purl = ? AND filename = ? `) - _, err := db.Exec(query, storagePath, contentHash, size, contentType, now.UTC(), now, versionPURL, filename) + _, err := db.Exec(query, storagePath, contentHash, size, contentType, now, now, versionPURL, filename) return err } @@ -570,9 +299,9 @@ func (db *DB) GetLeastRecentlyUsedArtifacts(limit int) ([]Artifact, error) { query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at, source + created_at, updated_at FROM artifacts - WHERE storage_path IS NOT NULL AND source <> 'local' + WHERE storage_path IS NOT NULL ORDER BY last_accessed_at ASC NULLS FIRST LIMIT ? `) @@ -585,7 +314,7 @@ func (db *DB) GetLeastRecentlyUsedArtifacts(limit int) ([]Artifact, error) { func (db *DB) GetTotalCacheSize() (int64, error) { var total sql.NullInt64 - err := db.Get(&total, `SELECT SUM(size) FROM artifacts WHERE storage_path IS NOT NULL AND source <> 'local'`) + err := db.Get(&total, `SELECT SUM(size) FROM artifacts WHERE storage_path IS NOT NULL`) if err != nil { return 0, err } @@ -601,84 +330,14 @@ func (db *DB) GetCachedArtifactCount() (int64, error) { return count, err } -// ClearArtifactCache marks an artifact uncached if its record still points at -// storagePath, queues storagePath for deletion in the same transaction, and -// reports whether it did. A record a newer fetch has moved elsewhere is left -// alone, so a clear never orphans the object that fetch committed. If -// queueing fails, the record keeps its path. -func (db *DB) ClearArtifactCache(versionPURL, filename, storagePath string) (bool, error) { - tx, err := db.Beginx() - if err != nil { - return false, err - } - defer func() { _ = tx.Rollback() }() - +func (db *DB) ClearArtifactCache(versionPURL, filename string) error { query := db.Rebind(` UPDATE artifacts SET storage_path = NULL, content_hash = NULL, size = NULL, content_type = NULL, fetched_at = NULL, updated_at = ? - WHERE version_purl = ? AND filename = ? AND storage_path = ? + WHERE version_purl = ? AND filename = ? `) - res, err := tx.Exec(query, time.Now(), versionPURL, filename, storagePath) - if err != nil { - return false, err - } - n, err := res.RowsAffected() - if err != nil || n == 0 { - return false, err - } - if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), storagePath, time.Now().UTC()); err != nil { - return false, fmt.Errorf("queueing %s for deletion: %w", storagePath, err) - } - return true, tx.Commit() -} - -// DiscardArtifact clears the record and queues storagePath as -// ClearArtifactCache does, for callers that must not delete an object another -// request may be reading. -func (db *DB) DiscardArtifact(versionPURL, filename, storagePath string) error { - _, err := db.ClearArtifactCache(versionPURL, filename, storagePath) - return err -} - -// QueuePendingDelete queues a storage path no record points at any more. -// Queueing a path again, as reclaim does after a failed delete, restarts its -// grace period and moves it behind the rest. -func (db *DB) QueuePendingDelete(path string) error { - _, err := db.Exec(db.Rebind(queuePendingDeleteQuery), path, time.Now().UTC()) - return err -} - -// queuePendingDeleteQuery queues a path, or moves an already queued one to -// the back of the queue. It takes the path and the time it is queued at. -const queuePendingDeleteQuery = ` - INSERT INTO pending_deletes (path, queued_at) VALUES (?, ?) - ON CONFLICT(path) DO UPDATE SET queued_at = excluded.queued_at` - -// GetDuePendingDeletes returns up to limit paths queued before cutoff, oldest -// first. A path a record points at again is skipped: a metadata document is -// stored at a path fixed by its key, so one fetched again after eviction -// reuses the path queued for it. -func (db *DB) GetDuePendingDeletes(cutoff time.Time, limit int) ([]string, error) { - var paths []string - query := db.Rebind(` - SELECT path FROM pending_deletes - WHERE queued_at < ? - AND NOT EXISTS (SELECT 1 FROM artifacts WHERE artifacts.storage_path = pending_deletes.path) - AND NOT EXISTS (SELECT 1 FROM metadata_cache WHERE metadata_cache.storage_path = pending_deletes.path) - AND NOT EXISTS (SELECT 1 FROM metadata_snapshots WHERE metadata_snapshots.storage_path = pending_deletes.path) - ORDER BY queued_at - LIMIT ? - `) - if err := db.Select(&paths, query, cutoff.UTC(), limit); err != nil { - return nil, err - } - return paths, nil -} - -// RemovePendingDelete drops a deleted path from the queue. -func (db *DB) RemovePendingDelete(path string) error { - _, err := db.Exec(db.Rebind(`DELETE FROM pending_deletes WHERE path = ?`), path) + _, err := db.Exec(query, time.Now(), versionPURL, filename) return err } @@ -784,14 +443,11 @@ func (db *DB) GetMostPopularPackages(limit int) ([]PopularPackage, error) { } type RecentPackage struct { - Ecosystem string `db:"ecosystem"` - Name string `db:"name"` - VersionPURL string `db:"version_purl"` - CachedAt time.Time `db:"fetched_at"` - Size int64 `db:"size"` - // Version is derived from VersionPURL rather than selected, so that the - // PURL percent-encoding is decoded (e.g. "%2B" back to "+"). - Version string `db:"-"` + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + Version string `db:"version"` + CachedAt time.Time `db:"fetched_at"` + Size int64 `db:"size"` } func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) { @@ -805,10 +461,10 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) { } var packages []RecentPackage - // There is no separate version column, so the full version PURL is selected - // and the version is decoded from it in Go. + // We need to extract version from the purl since there's no separate version column query := db.Rebind(` - SELECT p.ecosystem, p.name, v.purl as version_purl, + SELECT p.ecosystem, p.name, + SUBSTR(v.purl, INSTR(v.purl, '@') + 1) as version, a.fetched_at, COALESCE(a.size, 0) as size FROM artifacts a JOIN versions v ON v.purl = a.version_purl @@ -818,13 +474,25 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) { LIMIT ? `) + // For postgres, use different string function + if db.dialect == DialectPostgres { + query = db.Rebind(` + SELECT p.ecosystem, p.name, + SUBSTRING(v.purl FROM POSITION('@' IN v.purl) + 1) as version, + a.fetched_at, COALESCE(a.size, 0) as size + FROM artifacts a + JOIN versions v ON v.purl = a.version_purl + JOIN packages p ON p.purl = v.package_purl + WHERE a.storage_path IS NOT NULL AND a.fetched_at IS NOT NULL + ORDER BY a.fetched_at DESC + LIMIT ? + `) + } + err = db.Select(&packages, query, limit) if err != nil { return nil, err } - for i := range packages { - packages[i].Version = VersionFromPURL(packages[i].VersionPURL) - } return packages, nil } @@ -849,13 +517,12 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i } var results []SearchResult - searchPattern := nameSearchPattern(query) + searchPattern := "%" + query + "%" var sqlQuery string var args []any if ecosystem != "" { - condition, ecosystemArgs := ecosystemFilter(ecosystem) sqlQuery = db.Rebind(` SELECT p.ecosystem, p.name, p.latest_version, p.license, COALESCE(SUM(a.hit_count), 0) as hits, @@ -864,12 +531,12 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE ` + nameSearchCondition + ` AND ` + condition + ` AND a.storage_path IS NOT NULL + WHERE p.name LIKE ? AND p.ecosystem = ? AND a.storage_path IS NOT NULL GROUP BY p.purl, p.ecosystem, p.name, p.latest_version, p.license ORDER BY hits DESC LIMIT ? OFFSET ? `) - args = append(append([]any{searchPattern}, ecosystemArgs...), limit, offset) + args = []any{searchPattern, ecosystem, limit, offset} } else { sqlQuery = db.Rebind(` SELECT p.ecosystem, p.name, p.latest_version, p.license, @@ -879,7 +546,7 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE ` + nameSearchCondition + ` AND a.storage_path IS NOT NULL + WHERE p.name LIKE ? AND a.storage_path IS NOT NULL GROUP BY p.purl, p.ecosystem, p.name, p.latest_version, p.license ORDER BY hits DESC LIMIT ? OFFSET ? @@ -894,18 +561,6 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i return results, nil } -// nameSearchCondition matches package names containing a search pattern -// from nameSearchPattern, ignoring case on SQLite and Postgres alike. -const nameSearchCondition = `LOWER(p.name) LIKE LOWER(?) ESCAPE '!'` - -// nameSearchPattern is the LIKE pattern for names containing query. A % -// in query is matched literally, but _ stays a single-character wildcard -// so typing_extensions still finds typing-extensions. -func nameSearchPattern(query string) string { - escaped := strings.NewReplacer("!", "!!", "%", "!%").Replace(query) - return "%" + escaped + "%" -} - func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) { hasArtifacts, err := db.HasTable("artifacts") if err != nil { @@ -915,27 +570,26 @@ func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) return 0, nil } - searchPattern := nameSearchPattern(query) + searchPattern := "%" + query + "%" var sqlQuery string var args []any if ecosystem != "" { - condition, ecosystemArgs := ecosystemFilter(ecosystem) sqlQuery = db.Rebind(` SELECT COUNT(DISTINCT p.purl) FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE ` + nameSearchCondition + ` AND ` + condition + ` AND a.storage_path IS NOT NULL + WHERE p.name LIKE ? AND p.ecosystem = ? AND a.storage_path IS NOT NULL `) - args = append([]any{searchPattern}, ecosystemArgs...) + args = []any{searchPattern, ecosystem} } else { sqlQuery = db.Rebind(` SELECT COUNT(DISTINCT p.purl) FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE ` + nameSearchCondition + ` AND a.storage_path IS NOT NULL + WHERE p.name LIKE ? AND a.storage_path IS NOT NULL `) args = []any{searchPattern} } @@ -1167,9 +821,8 @@ func (db *DB) ListCachedPackages(ecosystem string, sortBy string, limit int, off whereClause := "WHERE a.storage_path IS NOT NULL" args := []any{} if ecosystem != "" { - condition, ecosystemArgs := ecosystemFilter(ecosystem) - whereClause += " AND " + condition - args = append(args, ecosystemArgs...) + whereClause += " AND p.ecosystem = ?" + args = append(args, ecosystem) } groupByClause := "GROUP BY p.purl, p.ecosystem, p.name, p.latest_version, p.license" @@ -1204,41 +857,6 @@ func (db *DB) ListCachedPackages(ecosystem string, sortBy string, limit int, off return packages, nil } -// CountCachedPackagesByEcosystem counts packages with at least one stored artifact. -func (db *DB) CountCachedPackagesByEcosystem() (map[string]int64, error) { - hasArtifacts, err := db.HasTable("artifacts") - if err != nil { - return nil, err - } - counts := make(map[string]int64) - if !hasArtifacts { - return counts, nil - } - - rows, err := db.Query(` - SELECT p.ecosystem, COUNT(DISTINCT p.purl) - FROM packages p - JOIN versions v ON v.package_purl = p.purl - JOIN artifacts a ON a.version_purl = v.purl - WHERE a.storage_path IS NOT NULL - GROUP BY p.ecosystem - `) - if err != nil { - return nil, err - } - defer func() { _ = rows.Close() }() - - for rows.Next() { - var ecosystem string - var count int64 - if err := rows.Scan(&ecosystem, &count); err != nil { - return nil, err - } - counts[ecosystem] = count - } - return counts, rows.Err() -} - func (db *DB) CountCachedPackages(ecosystem string) (int64, error) { hasArtifacts, err := db.HasTable("artifacts") if err != nil { @@ -1251,9 +869,8 @@ func (db *DB) CountCachedPackages(ecosystem string) (int64, error) { whereClause := "WHERE a.storage_path IS NOT NULL" args := []any{} if ecosystem != "" { - condition, ecosystemArgs := ecosystemFilter(ecosystem) - whereClause += " AND " + condition - args = append(args, ecosystemArgs...) + whereClause += " AND p.ecosystem = ?" + args = append(args, ecosystem) } query := fmt.Sprintf(` @@ -1273,67 +890,11 @@ func (db *DB) CountCachedPackages(ecosystem string) (int64, error) { // Metadata cache queries -// MetadataCacheStats is one ecosystem's share of the metadata cache. -type MetadataCacheStats struct { - Ecosystem string - Entries int64 - Size int64 -} - -// GetMetadataCacheStats returns the number and total size of cached metadata -// documents per ecosystem, largest first. Ecosystem names are normalized so -// aliases such as "gem" and "rubygems" share one row. Rows written before -// sizes were recorded count as entries with no size. -func (db *DB) GetMetadataCacheStats() ([]MetadataCacheStats, error) { - hasTable, err := db.HasTable("metadata_cache") - if err != nil || !hasTable { - return nil, err - } - - rows, err := db.Query(`SELECT ecosystem, COUNT(*), COALESCE(SUM(size), 0) FROM metadata_cache GROUP BY ecosystem`) - if err != nil { - return nil, err - } - defer func() { _ = rows.Close() }() - - byEcosystem := make(map[string]*MetadataCacheStats) - for rows.Next() { - var ecosystem string - var entries, size int64 - if err := rows.Scan(&ecosystem, &entries, &size); err != nil { - return nil, err - } - key := purl.NormalizeEcosystem(ecosystem) - row, ok := byEcosystem[key] - if !ok { - row = &MetadataCacheStats{Ecosystem: key} - byEcosystem[key] = row - } - row.Entries += entries - row.Size += size - } - if err := rows.Err(); err != nil { - return nil, err - } - - stats := make([]MetadataCacheStats, 0, len(byEcosystem)) - for _, row := range byEcosystem { - stats = append(stats, *row) - } - sort.Slice(stats, func(i, j int) bool { - if stats[i].Size != stats[j].Size { - return stats[i].Size > stats[j].Size - } - return stats[i].Ecosystem < stats[j].Ecosystem - }) - return stats, nil -} - func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, error) { var entry MetadataCacheEntry query := db.Rebind(` - SELECT id, ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at + SELECT id, ecosystem, name, storage_path, etag, content_type, + size, last_modified, fetched_at, created_at, updated_at FROM metadata_cache WHERE ecosystem = ? AND name = ? `) err := db.Get(&entry, query, ecosystem, name) @@ -1346,374 +907,46 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err return &entry, nil } -// RefreshMetadataCache sets fetched_at on the metadata cache row for ecosystem -// and name, provided the row still records digest, and reports whether it did. -// It is for an upstream 304, which leaves the stored document as it is. -func (db *DB) RefreshMetadataCache(ecosystem, name, digest string, fetchedAt time.Time) (bool, error) { - // fetched_at is stored in UTC, as in UpsertMetadataCache. - res, err := db.Exec(db.Rebind(` - UPDATE metadata_cache SET fetched_at = ?, updated_at = ? - WHERE ecosystem = ? AND name = ? AND content_digest = ? - `), fetchedAt.UTC(), time.Now(), ecosystem, name, digest) - if err != nil { - return false, err - } - n, err := res.RowsAffected() - return n > 0, err -} - -// MoveMetadataCache points the metadata cache row for ecosystem and name at -// to, provided it is still at from and records digest, and reports whether -// it did. The caller has copied the document from from to to. -func (db *DB) MoveMetadataCache(ecosystem, name, from, to, digest string) (bool, error) { - res, err := db.Exec(db.Rebind(` - UPDATE metadata_cache SET storage_path = ?, updated_at = ? - WHERE ecosystem = ? AND name = ? AND storage_path = ? AND content_digest = ? - `), to, time.Now(), ecosystem, name, from, digest) - if err != nil { - return false, err - } - n, err := res.RowsAffected() - return n > 0, err -} - -// ForgetMetadataDocument points the metadata cache row for ecosystem and name -// at to, provided it is still at from, and clears what described the document -// at from: its digest, validators and fetch time. The row stays, so the next -// lookup fetches the document again rather than serving or revalidating it. -func (db *DB) ForgetMetadataDocument(ecosystem, name, from, to string) error { - _, err := db.Exec(db.Rebind(` - UPDATE metadata_cache - SET storage_path = ?, content_digest = NULL, etag = NULL, last_modified = NULL, - fetched_at = NULL, updated_at = ? - WHERE ecosystem = ? AND name = ? AND storage_path = ? - `), to, time.Now(), ecosystem, name, from) - return err -} - -// expireMetadataBatch is how many names one ExpireMetadataCache statement -// covers, well below the bind parameter limits of SQLite and Postgres. -const expireMetadataBatch = 500 - -// ExpireMetadataCache clears fetched_at on the metadata cache rows for -// ecosystem and names that were fetched before cutoff, and returns the names -// of the rows it changed. The rows keep their documents and validators, so -// the next lookup of each revalidates it upstream instead of serving it as -// fresh. It is for an upstream that reports which documents changed. -func (db *DB) ExpireMetadataCache(ecosystem string, names []string, cutoff time.Time) ([]string, error) { - var expired []string - for batch := range slices.Chunk(names, expireMetadataBatch) { - args := []any{time.Now(), ecosystem, cutoff.UTC()} - for _, name := range batch { - args = append(args, name) - } - var got []string - err := db.DB.Select(&got, db.Rebind(` - UPDATE metadata_cache SET fetched_at = NULL, updated_at = ? - WHERE ecosystem = ? AND fetched_at < ? - AND name IN (`+strings.TrimSuffix(strings.Repeat("?, ", len(batch)), ", ")+`) - RETURNING name - `), args...) - if err != nil { - return expired, err - } - expired = append(expired, got...) - } - return expired, nil -} - -// ExpireAllMetadataCache is ExpireMetadataCache for every row of ecosystem, -// except those of packages routed to another registry (named "routes/..."), -// for an upstream that can't say which of its documents changed. -func (db *DB) ExpireAllMetadataCache(ecosystem string, cutoff time.Time) ([]string, error) { - var expired []string - err := db.DB.Select(&expired, db.Rebind(` - UPDATE metadata_cache SET fetched_at = NULL, updated_at = ? - WHERE ecosystem = ? AND fetched_at < ? AND name NOT LIKE 'routes/%' - RETURNING name - `), time.Now(), ecosystem, cutoff.UTC()) - return expired, err -} - -// UpsertMetadataCache writes entry and marks it accessed now, since a -// document is stored because a request asked for it. func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { now := time.Now() - // fetched_at is a TIMESTAMP without zone on Postgres, and lib/pq reads it - // back labelled UTC, so it has to be stored in UTC. last_modified already - // is, since it comes from an HTTP date. - fetchedAt := entry.FetchedAt - if fetchedAt.Valid { - fetchedAt.Time = fetchedAt.Time.UTC() - } var query string if db.dialect == DialectPostgres { query = ` - INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type, + size, last_modified, fetched_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = EXCLUDED.storage_path, etag = EXCLUDED.etag, - link = EXCLUDED.link, content_type = EXCLUDED.content_type, - content_encoding = EXCLUDED.content_encoding, - content_digest = EXCLUDED.content_digest, size = EXCLUDED.size, last_modified = EXCLUDED.last_modified, fetched_at = EXCLUDED.fetched_at, - last_accessed_at = EXCLUDED.last_accessed_at, updated_at = EXCLUDED.updated_at ` } else { query = ` - INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type, + size, last_modified, fetched_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = excluded.storage_path, etag = excluded.etag, - link = excluded.link, content_type = excluded.content_type, - content_encoding = excluded.content_encoding, - content_digest = excluded.content_digest, size = excluded.size, last_modified = excluded.last_modified, fetched_at = excluded.fetched_at, - last_accessed_at = excluded.last_accessed_at, updated_at = excluded.updated_at ` } _, err := db.Exec(query, - entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag, entry.Link, - entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.Size, entry.LastModified, fetchedAt, now.UTC(), now, now, + entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag, + entry.ContentType, entry.Size, entry.LastModified, entry.FetchedAt, now, now, ) if err != nil { return fmt.Errorf("upserting metadata cache: %w", err) } return nil } - -// Analytics queries - -// EcosystemStats aggregates cache and download activity for one ecosystem. -// -// DownloadedBytes is the accumulated download volume: every cache hit on an -// artifact served its full size, so the sum of hit_count * size is the number -// of bytes the proxy has handed to clients from cache for this ecosystem. -type EcosystemStats struct { - Ecosystem string `db:"ecosystem"` - Packages int64 `db:"packages"` - Versions int64 `db:"versions"` - Artifacts int64 `db:"artifacts"` - CacheSize int64 `db:"cache_size"` - Downloads int64 `db:"downloads"` - DownloadedBytes int64 `db:"downloaded_bytes"` -} - -// GetEcosystemStats returns per-ecosystem cache and download totals, ordered by -// accumulated download volume descending. Ecosystems with rows in packages but -// nothing cached are included with zeroed artifact counters. -// -// Artifacts evicted from the cache no longer contribute: eviction clears the -// size column, so their historical hits drop out of the accumulated total. -func (db *DB) GetEcosystemStats() ([]EcosystemStats, error) { - byEcosystem := make(map[string]*EcosystemStats) - - get := func(ecosystem string) *EcosystemStats { - if s, ok := byEcosystem[ecosystem]; ok { - return s - } - s := &EcosystemStats{Ecosystem: ecosystem} - byEcosystem[ecosystem] = s - return s - } - - if err := db.eachCount(`SELECT ecosystem, COUNT(*) FROM packages GROUP BY ecosystem`, - func(ecosystem string, n int64) { get(ecosystem).Packages = n }); err != nil { - return nil, err - } - - // Left joined for the same reason as artifacts below: a version whose - // package row is missing would otherwise vanish here while still counting - // in GetCacheStats' COUNT(*), leaving the two unable to reconcile. Such - // versions are mostly publish times cooldown stored for versions nobody - // downloaded, so they count under the type of their package PURL, and - // only a PURL that can't be parsed is unattributed. - if err := db.eachCount(` - SELECT COALESCE(p.ecosystem, '`+orphanVersionPrefix+`' || v.package_purl), COUNT(*) - FROM versions v - LEFT JOIN packages p ON p.purl = v.package_purl - GROUP BY COALESCE(p.ecosystem, '`+orphanVersionPrefix+`' || v.package_purl) - `, func(key string, n int64) { get(versionEcosystem(key)).Versions += n }); err != nil { - return nil, err - } - - // The artifacts table is proxy-specific: a database inherited from - // git-pkgs carries packages and versions without it. - hasArtifacts, err := db.HasTable("artifacts") - if err != nil { - return nil, err - } - if hasArtifacts { - if err := db.eachArtifactStat(get); err != nil { - return nil, err - } - } - - stats := make([]EcosystemStats, 0, len(byEcosystem)) - for _, s := range byEcosystem { - stats = append(stats, *s) - } - stats = mergeAliasedEcosystems(stats) - sortEcosystemStats(stats) - return stats, nil -} - -// mergeAliasedEcosystems combines rows whose ecosystem names normalize to the -// same canonical name. The proxy writes "gem" and git-pkgs writes "rubygems", -// so a database that has seen both carries two rows for one ecosystem; left -// split they would render as two table rows and two chart slices, each with -// half the real share. -// -// The surviving row keeps the raw spelling of whichever input held the most -// cached bytes, because that string is what the UI filters and links by: the -// packages table stores the raw value, so substituting the canonical name would -// produce links that match nothing. That comparison is against each input's own -// size, not the running total, which would otherwise let the first spelling win -// simply by being merged into first. -func mergeAliasedEcosystems(stats []EcosystemStats) []EcosystemStats { - merged := make(map[string]*EcosystemStats, len(stats)) - largest := make(map[string]int64, len(stats)) - order := make([]string, 0, len(stats)) - - for i := range stats { - key := purl.NormalizeEcosystem(stats[i].Ecosystem) - into, ok := merged[key] - if !ok { - row := stats[i] - merged[key] = &row - largest[key] = stats[i].CacheSize - order = append(order, key) - continue - } - - // The name tiebreak matters: GetEcosystemStats builds its input by - // ranging a map, so without it two spellings of equal size would swap - // between refreshes and flip the row's badge and filter link. - if stats[i].CacheSize > largest[key] || - (stats[i].CacheSize == largest[key] && stats[i].Ecosystem < into.Ecosystem) { - largest[key] = stats[i].CacheSize - into.Ecosystem = stats[i].Ecosystem - } - into.Packages += stats[i].Packages - into.Versions += stats[i].Versions - into.Artifacts += stats[i].Artifacts - into.CacheSize += stats[i].CacheSize - into.Downloads += stats[i].Downloads - into.DownloadedBytes += stats[i].DownloadedBytes - } - - out := make([]EcosystemStats, 0, len(order)) - for _, key := range order { - out = append(out, *merged[key]) - } - return out -} - -// unattributedEcosystem collects cached artifacts whose version or package row -// is missing. Nothing enforces that link at the schema level, so an inner join -// would silently drop such rows and leave the per-ecosystem totals short of -// GetTotalCacheSize — two numbers that sit side by side in the UI and in -// Grafana. Bucketing them keeps the two reconcilable. -const unattributedEcosystem = "unattributed" - -// orphanVersionPrefix marks the package PURL of a version without a package -// row in GetEcosystemStats. No ecosystem name starts with it. -const orphanVersionPrefix = "~" - -// versionEcosystem returns the ecosystem a GetEcosystemStats version count -// is for: the package row's, or the type of the package PURL of a version -// without one. The ecosystems that store publish times are named like their -// PURL types. -func versionEcosystem(key string) string { - packagePURL, orphan := strings.CutPrefix(key, orphanVersionPrefix) - if !orphan { - return key - } - p, err := purl.Parse(packagePURL) - if err != nil { - return unattributedEcosystem - } - return p.Type -} - -func (db *DB) eachArtifactStat(get func(string) *EcosystemStats) error { - rows, err := db.Query(` - SELECT COALESCE(p.ecosystem, '` + unattributedEcosystem + `'), - COUNT(*), - COALESCE(SUM(a.size), 0), - COALESCE(SUM(a.hit_count), 0), - COALESCE(SUM(a.hit_count * a.size), 0) - FROM artifacts a - LEFT JOIN versions v ON v.purl = a.version_purl - LEFT JOIN packages p ON p.purl = v.package_purl - WHERE a.storage_path IS NOT NULL - GROUP BY COALESCE(p.ecosystem, '` + unattributedEcosystem + `') - `) - if err != nil { - return err - } - defer func() { _ = rows.Close() }() - - for rows.Next() { - var ecosystem string - var artifacts, cacheSize, downloads, downloadedBytes int64 - if err := rows.Scan(&ecosystem, &artifacts, &cacheSize, &downloads, &downloadedBytes); err != nil { - return err - } - s := get(ecosystem) - s.Artifacts = artifacts - s.CacheSize = cacheSize - s.Downloads = downloads - s.DownloadedBytes = downloadedBytes - } - return rows.Err() -} - -// eachCount runs a two-column "group by" query and hands each (key, count) pair to fn. -func (db *DB) eachCount(query string, fn func(key string, n int64)) error { - rows, err := db.Query(query) - if err != nil { - return err - } - defer func() { _ = rows.Close() }() - - for rows.Next() { - var key string - var n int64 - if err := rows.Scan(&key, &n); err != nil { - return err - } - fn(key, n) - } - return rows.Err() -} - -// sortEcosystemStats orders by accumulated download volume, then by cache size, -// then by name, so that ecosystems with no traffic yet still sort predictably. -func sortEcosystemStats(stats []EcosystemStats) { - sort.Slice(stats, func(i, j int) bool { - a, b := stats[i], stats[j] - switch { - case a.DownloadedBytes != b.DownloadedBytes: - return a.DownloadedBytes > b.DownloadedBytes - case a.CacheSize != b.CacheSize: - return a.CacheSize > b.CacheSize - default: - return a.Ecosystem < b.Ecosystem - } - }) -} diff --git a/internal/database/queries_packages_list_test.go b/internal/database/queries_packages_list_test.go index 5eaa402..fc9356c 100644 --- a/internal/database/queries_packages_list_test.go +++ b/internal/database/queries_packages_list_test.go @@ -2,58 +2,12 @@ package database import ( "database/sql" - "maps" - "slices" "testing" "time" ) const testEcosystemNPM = "npm" -func TestCountCachedPackagesByEcosystem(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - counts, err := db.CountCachedPackagesByEcosystem() - if err != nil || len(counts) != 0 { - t.Fatalf("empty cache counts = %v, error = %v", counts, err) - } - - seedArtifact(t, db, "npm", "express", "1.0.0", 100, 1) - seedArtifact(t, db, "npm", "express", "2.0.0", 200, 1) - seedArtifact(t, db, "npm", "lodash", "1.0.0", 100, 1) - seedArtifact(t, db, "cargo", "serde", "1.0.0", 100, 1) - seedArtifact(t, db, "pypi", "evicted", "1.0.0", 100, 1) - if _, err := db.Exec(`UPDATE artifacts SET storage_path = NULL WHERE version_purl = 'pkg:pypi/evicted@1.0.0'`); err != nil { - t.Fatal(err) - } - if err := db.UpsertPackage(&Package{PURL: "pkg:gem/metadata-only", Ecosystem: "gem", Name: "metadata-only"}); err != nil { - t.Fatal(err) - } - if err := db.UpsertArtifact(&Artifact{ - VersionPURL: "pkg:npm/express@1.0.0", - Filename: "extra.tgz", - UpstreamURL: "https://example.test/extra.tgz", - StoragePath: sql.NullString{String: "extra.tgz", Valid: true}, - }); err != nil { - t.Fatal(err) - } - - counts, err = db.CountCachedPackagesByEcosystem() - if err != nil { - t.Fatal(err) - } - want := map[string]int64{"npm": 2, "cargo": 1} - if !maps.Equal(counts, want) { - t.Fatalf("counts = %v, want %v", counts, want) - } - for ecosystem, count := range counts { - listed, err := db.CountCachedPackages(ecosystem) - if err != nil || count != listed { - t.Errorf("%s count = %d, list count = %d, error = %v", ecosystem, count, listed, err) - } - } - }) -} - func setupListCachedPackagesDB(t *testing.T) *DB { t.Helper() @@ -219,76 +173,3 @@ func TestListCachedPackages(t *testing.T) { } }) } - -// git-pkgs, and proxy mirror before it used the handler names, wrote -// "rubygems" where the gem handler writes "gem". Filtering by either name -// finds both. -func TestCachedPackageFiltersMatchEcosystemAliases(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - seedArtifact(t, db, "rubygems", "rake", "13.2.1", 100, 1) - seedArtifact(t, db, "gem", "rails", "8.0.0", 100, 1) - seedArtifact(t, db, "npm", "express", "1.0.0", 100, 1) - - for _, ecosystem := range []string{"gem", "rubygems"} { - listed, err := db.ListCachedPackages(ecosystem, "name", 50, 0) - if err != nil { - t.Fatal(err) - } - var names []string - for _, p := range listed { - names = append(names, p.Name) - } - if len(names) != 2 || names[0] != "rails" || names[1] != "rake" { - t.Errorf("ListCachedPackages(%q) = %v, want rails and rake", ecosystem, names) - } - if count, err := db.CountCachedPackages(ecosystem); err != nil || count != 2 { - t.Errorf("CountCachedPackages(%q) = %d, %v, want 2", ecosystem, count, err) - } - found, err := db.SearchPackages("ra", ecosystem, 50, 0) - if err != nil || len(found) != 2 { - t.Errorf("SearchPackages(ra, %q) = %d results, %v, want 2", ecosystem, len(found), err) - } - } - }) -} - -func TestSearchPackagesIgnoresCaseAndEscapesPercent(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - seedArtifact(t, db, "pypi", "Django", "5.0.0", 100, 1) - seedArtifact(t, db, "pypi", "typing-extensions", "4.12.0", 100, 1) - seedArtifact(t, db, "npm", "my_pkg", "1.0.0", 100, 1) - seedArtifact(t, db, "npm", "100%-done", "1.0.0", 100, 1) - seedArtifact(t, db, "npm", "1000-done", "1.0.0", 100, 1) - seedArtifact(t, db, "npm", `back\slash`, "1.0.0", 100, 1) - - for _, tc := range []struct { - query string - want []string - }{ - {"django", []string{"Django"}}, - {"DJANGO", []string{"Django"}}, - {"my_pkg", []string{"my_pkg"}}, - // _ stays a single-character wildcard, so it matches - too. - {"typing_extensions", []string{"typing-extensions"}}, - {"typing-extensions", []string{"typing-extensions"}}, - {"100%", []string{"100%-done"}}, - {`k\s`, []string{`back\slash`}}, - } { - found, err := db.SearchPackages(tc.query, "", 50, 0) - if err != nil { - t.Fatalf("SearchPackages(%q): %v", tc.query, err) - } - var names []string - for _, p := range found { - names = append(names, p.Name) - } - if !slices.Equal(names, tc.want) { - t.Errorf("SearchPackages(%q) = %v, want %v", tc.query, names, tc.want) - } - count, err := db.CountSearchResults(tc.query, "") - if err != nil || count != int64(len(tc.want)) { - t.Errorf("CountSearchResults(%q) = %d, %v, want %d", tc.query, count, err, len(tc.want)) - } - } - }) -} diff --git a/internal/database/read_pool_bench_test.go b/internal/database/read_pool_bench_test.go deleted file mode 100644 index bae4cc5..0000000 --- a/internal/database/read_pool_bench_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package database - -import ( - "fmt" - "path/filepath" - "sync" - "sync/atomic" - "testing" - "time" -) - -// benchArtifacts is the artifact count the read pool benchmarks seed: about -// what a busy cache holds, and enough for the stats queries to take a while. -const ( - benchPackages = 30_000 - benchVersions = 100_000 - benchArtifacts = 300_000 -) - -// seedBenchDB creates a database at path with benchArtifacts artifacts -// spread over three ecosystems, written in one statement per table. -func seedBenchDB(b *testing.B, path string) { - b.Helper() - db, err := Create(path) - if err != nil { - b.Fatalf("Create: %v", err) - } - defer func() { _ = db.Close() }() - stmts := []string{ - fmt.Sprintf(`WITH RECURSIVE n(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM n WHERE i+1 < %d) - INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) - SELECT 'pkg:' || e || '/p' || i, e, 'p' || i, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP - FROM (SELECT i, CASE i %% 3 WHEN 0 THEN 'npm' WHEN 1 THEN 'pypi' ELSE 'cargo' END AS e FROM n)`, benchPackages), - fmt.Sprintf(`WITH RECURSIVE n(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM n WHERE i+1 < %d) - INSERT INTO versions (purl, package_purl, integrity, created_at, updated_at) - SELECT p.purl || '@' || n.i, p.purl, 'sha256-x', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP - FROM n JOIN packages p ON p.id = (n.i %% %d) + 1`, benchVersions, benchPackages), - fmt.Sprintf(`WITH RECURSIVE n(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM n WHERE i+1 < %d) - INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, size, - content_type, fetched_at, hit_count, last_accessed_at, created_at, updated_at) - SELECT v.purl, 'f' || n.i, 'https://example.org/' || n.i, 'blobs/' || n.i, 'h', 1000 + n.i, - 'application/octet-stream', CURRENT_TIMESTAMP, n.i %% 7, CURRENT_TIMESTAMP, - CURRENT_TIMESTAMP, CURRENT_TIMESTAMP - FROM n JOIN versions v ON v.id = (n.i %% %d) + 1`, benchArtifacts, benchVersions), - } - for _, s := range stmts { - if _, err := db.Exec(s); err != nil { - b.Fatalf("seeding: %v", err) - } - } -} - -// openBenchDB opens path. With oneConn it routes reads to the writer, as -// before the read pool existed. -func openBenchDB(b *testing.B, path string, oneConn bool) *DB { - b.Helper() - db, err := Open(path) - if err != nil { - b.Fatalf("Open: %v", err) - } - if oneConn { - _ = db.reader.Close() - db.reader = db.DB - } - b.Cleanup(func() { _ = db.Close() }) - return db -} - -// benchKey returns the lookup arguments for artifact i, as seeded above. -func benchKey(i int) (pkgPURL, versionPURL, filename string) { - v := i % benchVersions - p := v % benchPackages - eco := [...]string{"npm", "pypi", "cargo"}[p%3] - pkgPURL = fmt.Sprintf("pkg:%s/p%d", eco, p) - return pkgPURL, fmt.Sprintf("%s@%d", pkgPURL, v), fmt.Sprintf("f%d", i) -} - -func lookup(b *testing.B, db *DB, i int) { - pkgPURL, versionPURL, filename := benchKey(i % benchArtifacts) - art, err := db.GetCachedArtifact(pkgPURL, versionPURL, filename) - if err != nil || art == nil { - b.Fatalf("GetCachedArtifact(%s, %s) = %v, %v", versionPURL, filename, art, err) - } -} - -var poolModes = []struct { - name string - oneConn bool -}{ - {"one-connection", true}, - {"read-pool", false}, -} - -// BenchmarkSQLiteParallelLookups runs GetCachedArtifact from parallel -// goroutines, as concurrent downloads do. -func BenchmarkSQLiteParallelLookups(b *testing.B) { - path := filepath.Join(b.TempDir(), "bench.db") - seedBenchDB(b, path) - for _, mode := range poolModes { - b.Run(mode.name, func(b *testing.B) { - db := openBenchDB(b, path, mode.oneConn) - var next atomic.Int64 - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - lookup(b, db, int(next.Add(7919))) - } - }) - }) - } -} - -// BenchmarkSQLiteLookupDuringStats runs GetCachedArtifact one at a time -// while another goroutine keeps running GetEcosystemStats and GetCacheStats, -// as the metrics updater and the dashboard do. It reports the slowest lookup. -func BenchmarkSQLiteLookupDuringStats(b *testing.B) { - path := filepath.Join(b.TempDir(), "bench.db") - seedBenchDB(b, path) - for _, mode := range poolModes { - b.Run(mode.name, func(b *testing.B) { - db := openBenchDB(b, path, mode.oneConn) - stop := make(chan struct{}) - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - for { - select { - case <-stop: - return - default: - } - if _, err := db.GetEcosystemStats(); err != nil { - b.Errorf("GetEcosystemStats: %v", err) - return - } - if _, err := db.GetCacheStats(); err != nil { - b.Errorf("GetCacheStats: %v", err) - return - } - } - }() - - var slowest time.Duration - b.ResetTimer() - for i := range b.N { - start := time.Now() - lookup(b, db, i*7919) - slowest = max(slowest, time.Since(start)) - } - b.StopTimer() - close(stop) - wg.Wait() - b.ReportMetric(float64(slowest.Microseconds())/1000, "max-ms") - }) - } -} diff --git a/internal/database/read_pool_test.go b/internal/database/read_pool_test.go deleted file mode 100644 index 20436e1..0000000 --- a/internal/database/read_pool_test.go +++ /dev/null @@ -1,393 +0,0 @@ -package database - -import ( - "context" - "database/sql" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - "testing" - "time" -) - -func seedOnePackage(t *testing.T, db *DB) { - t.Helper() - pkg := &Package{PURL: "pkg:npm/left-pad", Ecosystem: "npm", Name: "left-pad"} - ver := &Version{PURL: "pkg:npm/left-pad@1.3.0", PackagePURL: pkg.PURL} - art := &Artifact{ - VersionPURL: ver.PURL, - Filename: "left-pad-1.3.0.tgz", - UpstreamURL: "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", - StoragePath: sql.NullString{String: "npm/left-pad/1.3.0/left-pad-1.3.0.tgz", Valid: true}, - Size: sql.NullInt64{Int64: 10, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - } - if err := db.RecordCachedArtifact(pkg, ver, art); err != nil { - t.Fatalf("RecordCachedArtifact: %v", err) - } -} - -func TestSQLiteUsesSeparateReadPool(t *testing.T) { - db := newTestDB(t) - if db.reader == db.DB { - t.Fatal("file database shares one pool for reads and writes") - } - if got, want := db.reader.Stats().MaxOpenConnections, sqliteReaderConns(); got != want { - t.Errorf("reader MaxOpenConnections = %d, want %d", got, want) - } - if got := db.DB.Stats().MaxOpenConnections; got != 1 { - t.Errorf("writer MaxOpenConnections = %d, want 1", got) - } -} - -func TestSQLiteReadPoolRejectsWrites(t *testing.T) { - db := newTestDB(t) - - if _, err := db.reader.Exec(`INSERT INTO pending_deletes (path, queued_at) VALUES ('x', CURRENT_TIMESTAMP)`); err == nil { - t.Fatal("write on the reader pool succeeded") - } else if !strings.Contains(err.Error(), "readonly") { - t.Fatalf("write on the reader pool failed with %v, want a read-only error", err) - } - - // Get goes to the reader, so a write sent through it must fail too - // rather than run. - var path string - err := db.Get(&path, `INSERT INTO pending_deletes (path, queued_at) VALUES ('y', CURRENT_TIMESTAMP) RETURNING path`) - if err == nil { - t.Fatal("INSERT ... RETURNING through Get succeeded") - } - - var n int - if err := db.Get(&n, `SELECT COUNT(*) FROM pending_deletes`); err != nil { - t.Fatalf("count: %v", err) - } - if n != 0 { - t.Fatalf("pending_deletes has %d rows after rejected writes, want 0", n) - } -} - -// pragmaInt reads an integer pragma on conn. -func pragmaInt(t *testing.T, conn *sql.Conn, name string) int { - t.Helper() - var v int - if err := conn.QueryRowContext(context.Background(), "PRAGMA "+name).Scan(&v); err != nil { - t.Fatalf("PRAGMA %s: %v", name, err) - } - return v -} - -func pragmaString(t *testing.T, conn *sql.Conn, name string) string { - t.Helper() - var v string - if err := conn.QueryRowContext(context.Background(), "PRAGMA "+name).Scan(&v); err != nil { - t.Fatalf("PRAGMA %s: %v", name, err) - } - return v -} - -// TestSQLitePragmasOnEveryConnection holds every connection of both pools at -// once, so each one is a separate SQLite connection, and checks that each -// got the settings from the DSN. -func TestSQLitePragmasOnEveryConnection(t *testing.T) { - db := newTestDB(t) - ctx := context.Background() - - writer, err := db.Conn(ctx) - if err != nil { - t.Fatalf("writer conn: %v", err) - } - defer func() { _ = writer.Close() }() - if got := pragmaInt(t, writer, "busy_timeout"); got != 5000 { - t.Errorf("writer busy_timeout = %d, want 5000", got) - } - if got := pragmaString(t, writer, "journal_mode"); got != "wal" { - t.Errorf("writer journal_mode = %q, want wal", got) - } - if got := pragmaInt(t, writer, "synchronous"); got != 1 { - t.Errorf("writer synchronous = %d, want 1 (NORMAL)", got) - } - if got := pragmaInt(t, writer, "query_only"); got != 0 { - t.Errorf("writer query_only = %d, want 0", got) - } - - n := sqliteReaderConns() - conns := make([]*sql.Conn, 0, n) - defer func() { - for _, c := range conns { - _ = c.Close() - } - }() - for i := range n { - c, err := db.reader.Conn(ctx) - if err != nil { - t.Fatalf("reader conn %d: %v", i, err) - } - conns = append(conns, c) - if got := pragmaInt(t, c, "busy_timeout"); got != 5000 { - t.Errorf("reader %d busy_timeout = %d, want 5000", i, got) - } - if got := pragmaString(t, c, "journal_mode"); got != "wal" { - t.Errorf("reader %d journal_mode = %q, want wal", i, got) - } - if got := pragmaInt(t, c, "query_only"); got != 1 { - t.Errorf("reader %d query_only = %d, want 1", i, got) - } - } - if got := db.reader.Stats().OpenConnections; got != n { - t.Errorf("reader pool has %d open connections, want %d", got, n) - } -} - -// TestSQLiteReadNotBlockedBySlowRead holds one reader connection in an open -// read, as a long stats scan does, and a write transaction on the writer. -// A lookup must still complete, and must not see the uncommitted write. -func TestSQLiteReadNotBlockedBySlowRead(t *testing.T) { - db := newTestDB(t) - seedOnePackage(t, db) - - rows, err := db.Query(`SELECT purl FROM packages`) - if err != nil { - t.Fatalf("Query: %v", err) - } - defer func() { _ = rows.Close() }() - if !rows.Next() { - t.Fatal("no package row") - } - - // Bounded, so with a single shared connection the test fails instead - // of waiting forever for the connection the open read holds. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - tx, err := db.BeginTxx(ctx, nil) - if err != nil { - t.Fatalf("BeginTxx while a read is open: %v", err) - } - defer func() { _ = tx.Rollback() }() - if _, err := tx.Exec(`INSERT INTO pending_deletes (path, queued_at) VALUES ('uncommitted', CURRENT_TIMESTAMP)`); err != nil { - t.Fatalf("insert in tx: %v", err) - } - - type result struct { - art *CachedArtifact - pending int - err error - } - done := make(chan result, 1) - go func() { - var r result - r.art, r.err = db.GetCachedArtifact("pkg:npm/left-pad", "pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz") - if r.err == nil { - r.err = db.GetContext(ctx, &r.pending, `SELECT COUNT(*) FROM pending_deletes`) - } - done <- r - }() - - select { - case r := <-done: - if r.err != nil { - t.Fatalf("lookup: %v", r.err) - } - if r.art == nil { - t.Fatal("lookup found no artifact") - } - if r.pending != 0 { - t.Errorf("reader saw %d uncommitted pending deletes, want 0", r.pending) - } - case <-time.After(5 * time.Second): - t.Fatal("lookup blocked behind an open read and a write transaction") - } - - if err := tx.Commit(); err != nil { - t.Fatalf("Commit: %v", err) - } - var pending int - if err := db.Get(&pending, `SELECT COUNT(*) FROM pending_deletes`); err != nil { - t.Fatalf("count after commit: %v", err) - } - if pending != 1 { - t.Errorf("reader saw %d pending deletes after commit, want 1", pending) - } -} - -// TestSQLiteReadAfterWrite checks that a read right after a write, on the -// other pool, sees it. -func TestSQLiteReadAfterWrite(t *testing.T) { - db := newTestDB(t) - for i := range 50 { - path := fmt.Sprintf("npm/p/%d.tgz", i) - if err := db.QueuePendingDelete(path); err != nil { - t.Fatalf("QueuePendingDelete: %v", err) - } - var n int - if err := db.Get(&n, db.Rebind(`SELECT COUNT(*) FROM pending_deletes WHERE path = ?`), path); err != nil { - t.Fatalf("count: %v", err) - } - if n != 1 { - t.Fatalf("write %d not visible to the reader", i) - } - } -} - -func TestSQLiteMemoryDatabaseSharesOneConnection(t *testing.T) { - db, err := Open(":memory:") - if err != nil { - t.Fatalf("Open: %v", err) - } - defer func() { _ = db.Close() }() - if db.reader != db.DB { - t.Fatal("in-memory database got a separate reader pool") - } - if err := db.CreateSchema(); err != nil { - t.Fatalf("CreateSchema: %v", err) - } - seedOnePackage(t, db) - art, err := db.GetCachedArtifact("pkg:npm/left-pad", "pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz") - if err != nil || art == nil { - t.Fatalf("GetCachedArtifact = %v, %v", art, err) - } -} - -func TestSQLiteCloseClosesBothPools(t *testing.T) { - db, err := Create(filepath.Join(t.TempDir(), "close.db")) - if err != nil { - t.Fatalf("Create: %v", err) - } - reader := db.reader - if err := db.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - if err := reader.Ping(); err == nil { - t.Fatal("reader pool still open after Close") - } - if err := db.Ping(); err == nil { - t.Fatal("writer pool still open after Close") - } -} - -func TestPostgresSharesOnePool(t *testing.T) { - url := os.Getenv("PROXY_DATABASE_URL") - if url == "" { - t.Skip("PROXY_DATABASE_URL not set, skipping postgres test") - } - db, err := OpenPostgres(url) - if err != nil { - t.Fatalf("OpenPostgres: %v", err) - } - defer func() { _ = db.Close() }() - if db.reader != db.DB { - t.Fatal("postgres got a separate reader pool") - } -} - -func TestIsMemoryPath(t *testing.T) { - for path, want := range map[string]bool{ - "": true, - ":memory:": true, - "file::memory:": true, - "file::memory:?cache=shared": true, - "file:cache?mode=memory": true, - "/var/lib/proxy/cache.db": false, - "cache.db": false, - } { - if got := isMemoryPath(path); got != want { - t.Errorf("isMemoryPath(%q) = %v, want %v", path, got, want) - } - } -} - -// TestSQLiteExistingRollbackJournalDatabase opens a database file left in -// rollback-journal mode, as an older or foreign tool may leave it. The -// writer must switch it to WAL before readers open it, or reads would block -// on writes. -func TestSQLiteExistingRollbackJournalDatabase(t *testing.T) { - path := filepath.Join(t.TempDir(), "old.db") - db, err := Create(path) - if err != nil { - t.Fatalf("Create: %v", err) - } - seedOnePackage(t, db) - if _, err := db.Exec("PRAGMA journal_mode = DELETE"); err != nil { - t.Fatalf("leaving WAL: %v", err) - } - if err := db.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - - db, err = Open(path) - if err != nil { - t.Fatalf("Open: %v", err) - } - defer func() { _ = db.Close() }() - var mode string - if err := db.reader.Get(&mode, "PRAGMA journal_mode"); err != nil || mode != "wal" { - t.Fatalf("reader journal_mode = %q, %v; want wal", mode, err) - } - art, err := db.GetCachedArtifact("pkg:npm/left-pad", "pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz") - if err != nil || art == nil { - t.Fatalf("GetCachedArtifact = %v, %v", art, err) - } -} - -// TestSQLiteConcurrentReadsAndWrites runs the writes a busy proxy makes -// (recording artifacts and hits) alongside request lookups and stats scans. -// None may fail with a busy, locked or read-only error. -func TestSQLiteConcurrentReadsAndWrites(t *testing.T) { - db := newTestDB(t) - seedOnePackage(t, db) - deadline := time.Now().Add(time.Second) - errs := make(chan error, 64) - var wg sync.WaitGroup - run := func(f func(i int) error) { - wg.Add(1) - go func() { - defer wg.Done() - for i := 0; time.Now().Before(deadline); i++ { - if err := f(i); err != nil { - errs <- err - return - } - } - }() - } - for w := range 2 { - run(func(i int) error { - name := fmt.Sprintf("pkg-%d-%d", w, i) - pkg := &Package{PURL: "pkg:npm/" + name, Ecosystem: "npm", Name: name} - ver := &Version{PURL: pkg.PURL + "@1.0.0", PackagePURL: pkg.PURL} - art := &Artifact{ - VersionPURL: ver.PURL, - Filename: name + ".tgz", - UpstreamURL: "https://registry.npmjs.org/" + name, - StoragePath: sql.NullString{String: "npm/" + name, Valid: true}, - Size: sql.NullInt64{Int64: 10, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - } - return db.RecordCachedArtifact(pkg, ver, art) - }) - } - run(func(int) error { return db.RecordArtifactHit("pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz") }) - for range 4 { - run(func(int) error { - art, err := db.GetCachedArtifact("pkg:npm/left-pad", "pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz") - if err == nil && art == nil { - err = errors.New("seeded artifact not found") - } - return err - }) - } - run(func(int) error { - if _, err := db.GetCacheStats(); err != nil { - return err - } - _, err := db.GetEcosystemStats() - return err - }) - wg.Wait() - close(errs) - for err := range errs { - t.Error(err) - } -} diff --git a/internal/database/rekey_artifact.go b/internal/database/rekey_artifact.go deleted file mode 100644 index 68a8dbd..0000000 --- a/internal/database/rekey_artifact.go +++ /dev/null @@ -1,40 +0,0 @@ -package database - -import ( - "fmt" - "time" -) - -// RekeyCachedArtifact moves one unchanged upstream record without replacing -// an existing destination. Access history and stored bytes are preserved. -func (db *DB) RekeyCachedArtifact(old *Artifact, pkg *Package, version *Version) (bool, error) { - tx, err := db.Beginx() - if err != nil { - return false, err - } - defer func() { _ = tx.Rollback() }() - now := time.Now() - _, err = tx.Exec(db.Rebind(`INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) ON CONFLICT(purl) DO NOTHING`), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now) - if err != nil { - return false, fmt.Errorf("creating artifact package: %w", err) - } - _, err = tx.Exec(db.Rebind(`INSERT INTO versions (purl, package_purl, created_at, updated_at) - VALUES (?, ?, ?, ?) ON CONFLICT(purl) DO NOTHING`), version.PURL, version.PackagePURL, now, now) - if err != nil { - return false, fmt.Errorf("creating artifact version: %w", err) - } - result, err := tx.Exec(db.Rebind(`UPDATE artifacts SET version_purl = ?, updated_at = ? - WHERE id = ? AND version_purl = ? AND storage_path = ? AND upstream_url = ? AND source = ? - AND NOT EXISTS (SELECT 1 FROM artifacts WHERE version_purl = ? AND filename = ?)`), - version.PURL, now, old.ID, old.VersionPURL, old.StoragePath, old.UpstreamURL, SourceUpstream, - version.PURL, old.Filename) - if err != nil { - return false, fmt.Errorf("moving cached artifact: %w", err) - } - count, err := result.RowsAffected() - if err != nil || count == 0 { - return false, err - } - return true, tx.Commit() -} diff --git a/internal/database/rekey_artifact_test.go b/internal/database/rekey_artifact_test.go deleted file mode 100644 index d5fd270..0000000 --- a/internal/database/rekey_artifact_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package database - -import ( - "database/sql" - "testing" -) - -func TestRekeyCachedArtifact(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - for _, scenario := range []string{"move", "destination exists", "source changed", "local source"} { - t.Run(scenario, func(t *testing.T) { - oldPackage := &Package{PURL: "pkg:golang/example.com/old", Ecosystem: "golang", Name: "example.com/Old"} - oldVersion := &Version{PURL: oldPackage.PURL + "@v1.0.0", PackagePURL: oldPackage.PURL} - old := &Artifact{VersionPURL: oldVersion.PURL, Filename: scenario + ".zip", - UpstreamURL: "https://proxy.example/example.com/!old/@v/v1.0.0.zip", - StoragePath: sql.NullString{String: scenario + "/old.zip", Valid: true}} - if err := db.RecordCachedArtifact(oldPackage, oldVersion, old); err != nil { - t.Fatal(err) - } - old, err := db.GetArtifact(oldVersion.PURL, old.Filename) - if err != nil || old == nil { - t.Fatalf("source artifact = %+v, %v", old, err) - } - pkg := &Package{PURL: "pkg:golang/example.com/Old", Ecosystem: "golang", Name: "example.com/Old"} - version := &Version{PURL: pkg.PURL + "@v1.0.0", PackagePURL: pkg.PURL} - prepareRekeyConflict(t, db, scenario, old, pkg, version) - moved, err := db.RekeyCachedArtifact(old, pkg, version) - if err != nil || moved != (scenario == "move") { - t.Fatalf("RekeyCachedArtifact = %v, %v", moved, err) - } - checkRekeyResult(t, db, scenario, old, version) - }) - } - }) -} - -func checkRekeyResult(t *testing.T, db *DB, scenario string, old *Artifact, version *Version) { - t.Helper() - current, err := db.GetArtifact(version.PURL, old.Filename) - if err != nil { - t.Fatal(err) - } - if scenario == "move" && (current == nil || current.ID != old.ID || current.StoragePath != old.StoragePath) { - t.Errorf("moved artifact = %+v, old = %+v", current, old) - } - if scenario == "destination exists" && (current == nil || current.StoragePath.String != "destination.zip") { - t.Errorf("destination replaced: %+v", current) - } - if paths := allPendingDeletes(t, db); len(paths) != 0 { - t.Errorf("rekey queued stored bytes for deletion: %v", paths) - } -} - -func prepareRekeyConflict(t *testing.T, db *DB, scenario string, old *Artifact, pkg *Package, version *Version) { - t.Helper() - var err error - switch scenario { - case "destination exists": - err = db.RecordCachedArtifact(pkg, version, &Artifact{VersionPURL: version.PURL, Filename: old.Filename, - StoragePath: sql.NullString{String: "destination.zip", Valid: true}}) - case "source changed": - _, err = db.Exec(db.Rebind(`UPDATE artifacts SET storage_path = ? WHERE id = ?`), "replacement.zip", old.ID) - case "local source": - _, err = db.Exec(db.Rebind(`UPDATE artifacts SET source = ? WHERE id = ?`), SourceLocal, old.ID) - } - if err != nil { - t.Fatal(err) - } -} diff --git a/internal/database/retention.go b/internal/database/retention.go deleted file mode 100644 index 0036362..0000000 --- a/internal/database/retention.go +++ /dev/null @@ -1,144 +0,0 @@ -package database - -import ( - "database/sql" - "errors" - "fmt" - "time" -) - -// RetentionCandidate is a cached artifact that has not been accessed since a -// cutoff, together with the package record it belongs to. -type RetentionCandidate struct { - ID int64 `db:"id"` - VersionPURL string `db:"version_purl"` - Filename string `db:"filename"` - StoragePath string `db:"storage_path"` - Size sql.NullInt64 `db:"size"` - FetchedAt sql.NullTime `db:"fetched_at"` - LastAccessedAt sql.NullTime `db:"last_accessed_at"` - MirroredAt sql.NullTime `db:"mirrored_at"` - Ecosystem string `db:"ecosystem"` - PackagePURL string `db:"package_purl"` -} - -// ExpiredBefore reports whether the artifact was neither fetched, accessed -// nor found by a mirror run at or after cutoff. Fetch and access are both -// checked: clearing a record keeps its last access time, so a record fetched -// again after an eviction has a fresh fetch time next to an old access time. -func (c RetentionCandidate) ExpiredBefore(cutoff time.Time) bool { - if !c.FetchedAt.Valid || !c.FetchedAt.Time.Before(cutoff) { - return false - } - if c.MirroredAt.Valid && !c.MirroredAt.Time.Before(cutoff) { - return false - } - return !c.LastAccessedAt.Valid || c.LastAccessedAt.Time.Before(cutoff) -} - -// expiredCondition selects cached artifacts fetched before the cutoff and -// neither accessed nor mirrored since. It takes the cutoff three times. -const expiredCondition = ` - storage_path IS NOT NULL - AND source <> 'local' - AND fetched_at < ? - AND (last_accessed_at IS NULL OR last_accessed_at < ?) - AND (mirrored_at IS NULL OR mirrored_at < ?)` - -// MaxArtifactID returns the highest artifact id, or zero for an empty table. -func (db *DB) MaxArtifactID() (int64, error) { - var maxID sql.NullInt64 - if err := db.Get(&maxID, `SELECT MAX(id) FROM artifacts`); err != nil { - return 0, err - } - return maxID.Int64, nil -} - -// GetRetentionCandidates returns the cached artifacts with fromID < id <= toID -// that expired before cutoff. Bounding the scan by id keeps each query short, -// which matters on SQLite, where every request shares one connection. -func (db *DB) GetRetentionCandidates(fromID, toID int64, cutoff time.Time) ([]RetentionCandidate, error) { - var candidates []RetentionCandidate - query := db.Rebind(` - SELECT a.id, a.version_purl, a.filename, a.storage_path, a.size, - a.fetched_at, a.last_accessed_at, a.mirrored_at, p.ecosystem, p.purl AS package_purl - FROM ( - SELECT * FROM artifacts - WHERE id > ? AND id <= ? AND` + expiredCondition + ` - ) a - JOIN versions v ON v.purl = a.version_purl - JOIN packages p ON p.purl = v.package_purl - ORDER BY a.id - `) - if err := db.Select(&candidates, query, fromID, toID, cutoff.UTC(), cutoff.UTC(), cutoff.UTC()); err != nil { - return nil, err - } - return candidates, nil -} - -// ClearExpiredArtifact marks an artifact uncached like ClearArtifactCache, -// but only while it is still expired before cutoff, so an artifact served -// between the scan and the clear stays cached. It queues storagePath for -// deletion in the same transaction: if queueing fails, the record keeps its -// path and a later sweep tries again, instead of the object staying in -// storage with nothing left pointing at it. -func (db *DB) ClearExpiredArtifact(versionPURL, filename, storagePath string, cutoff time.Time) (bool, error) { - tx, err := db.Beginx() - if err != nil { - return false, err - } - defer func() { _ = tx.Rollback() }() - - query := db.Rebind(` - UPDATE artifacts - SET storage_path = NULL, content_hash = NULL, size = NULL, - content_type = NULL, fetched_at = NULL, updated_at = ? - WHERE version_purl = ? AND filename = ? AND storage_path = ? AND` + expiredCondition) - res, err := tx.Exec(query, time.Now(), versionPURL, filename, storagePath, cutoff.UTC(), cutoff.UTC(), cutoff.UTC()) - if err != nil { - return false, err - } - n, err := res.RowsAffected() - if err != nil || n == 0 { - return false, err - } - if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), storagePath, time.Now().UTC()); err != nil { - return false, fmt.Errorf("queueing %s for deletion: %w", storagePath, err) - } - return true, tx.Commit() -} - -// MarkArtifactMirrored records that a mirror run found the artifact cached, -// which keeps it from retention without counting as a cache hit. -func (db *DB) MarkArtifactMirrored(versionPURL, filename string) error { - _, err := db.Exec(db.Rebind(` - UPDATE artifacts SET mirrored_at = ? - WHERE version_purl = ? AND filename = ? AND storage_path IS NOT NULL - `), time.Now().UTC(), versionPURL, filename) - return err -} - -// GetArtifactEcosystem returns the packages.ecosystem value of the package an -// artifact version belongs to, or "" when it has no package record. -func (db *DB) GetArtifactEcosystem(versionPURL string) (string, error) { - var ecosystem string - query := db.Rebind(` - SELECT p.ecosystem FROM versions v - JOIN packages p ON p.purl = v.package_purl - WHERE v.purl = ? - `) - err := db.Get(&ecosystem, query, versionPURL) - if errors.Is(err, sql.ErrNoRows) { - return "", nil - } - return ecosystem, err -} - -// FlushHits writes hits buffered by BatchHits now, so a caller about to read -// last_accessed_at sees them. -func (db *DB) FlushHits() error { - if db.hits == nil { - return nil - } - return db.flushHits() -} diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go deleted file mode 100644 index 2382011..0000000 --- a/internal/database/retention_test.go +++ /dev/null @@ -1,431 +0,0 @@ -package database - -import ( - "database/sql" - "slices" - "strings" - "testing" - "time" -) - -// seedRetentionArtifact stores a cached artifact for pkg:{ecosystem}/{name} -// with the given fetch and last access times; a zero accessed leaves the -// artifact never served. -// Artifacts store fetched_at and last_accessed_at in UTC, and the retention -// cutoffs are compared in UTC. In a zone east of UTC, any write or cutoff -// left in local time shifts an artifact by the offset on Postgres. -func TestArtifactTimesAreStoredInUTC(t *testing.T) { - tokyo, err := time.LoadLocation("Asia/Tokyo") - if err != nil { - t.Skipf("time zone data unavailable: %v", err) - } - prev := time.Local - time.Local = tokyo - t.Cleanup(func() { time.Local = prev }) - - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now() - old := now.Add(-48 * time.Hour) - seedRetentionArtifact(t, db, "npm", "stale", now.Add(-90*time.Minute), time.Time{}) - seedRetentionArtifact(t, db, "npm", "fresh", now.Add(-30*time.Minute), time.Time{}) - seedRetentionArtifact(t, db, "npm", "hit", old, time.Time{}) - if err := db.RecordArtifactHit("pkg:npm/hit@1.0.0", "hit.tgz"); err != nil { - t.Fatalf("RecordArtifactHit: %v", err) - } - seedRetentionArtifact(t, db, "npm", "batched", old, time.Time{}) - hits := map[hitKey]hitEntry{{"pkg:npm/batched@1.0.0", "batched.tgz"}: {count: 1, last: now.Add(-5 * time.Hour)}} - if err := db.writeHits(hits); err != nil { - t.Fatalf("writeHits: %v", err) - } - seedRetentionArtifact(t, db, "npm", "refetched", old, time.Time{}) - if err := db.MarkArtifactCached("pkg:npm/refetched@1.0.0", "refetched.tgz", "npm/refetched/1.0.0/refetched.tgz", "hash", 100, ""); err != nil { - t.Fatalf("MarkArtifactCached: %v", err) - } - - expired := func(cutoff time.Time) []string { - t.Helper() - candidates, err := db.GetRetentionCandidates(0, 100, cutoff) - if err != nil { - t.Fatalf("GetRetentionCandidates: %v", err) - } - var names []string - for _, c := range candidates { - if !c.ExpiredBefore(cutoff) { - t.Errorf("%s returned as a candidate but read back as not expired before %v", c.Filename, cutoff) - } - names = append(names, strings.TrimSuffix(c.Filename, ".tgz")) - } - slices.Sort(names) - return names - } - if got, want := expired(now.Add(-time.Hour)), []string{"batched", "stale"}; !slices.Equal(got, want) { - t.Errorf("expired an hour ago = %v, want %v", got, want) - } - if got, want := expired(now.Add(time.Hour)), []string{"batched", "fresh", "hit", "refetched", "stale"}; !slices.Equal(got, want) { - t.Errorf("expired an hour from now = %v, want %v", got, want) - } - cleared, err := db.ClearExpiredArtifact("pkg:npm/fresh@1.0.0", "fresh.tgz", "npm/fresh/1.0.0/fresh.tgz", now.Add(-time.Hour)) - if err != nil || cleared { - t.Errorf("ClearExpiredArtifact(fresh, an hour ago) = %v, %v, want false, nil", cleared, err) - } - }) -} - -func seedRetentionArtifact(t *testing.T, db *DB, ecosystem, name string, fetched, accessed time.Time) { - t.Helper() - pkgPURL := "pkg:" + ecosystem + "/" + name - versionPURL := pkgPURL + "@1.0.0" - if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: name}); err != nil { - t.Fatalf("UpsertPackage: %v", err) - } - if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - art := &Artifact{ - VersionPURL: versionPURL, - Filename: name + ".tgz", - UpstreamURL: "https://example.com/" + name + ".tgz", - StoragePath: sql.NullString{String: ecosystem + "/" + name + "/1.0.0/" + name + ".tgz", Valid: true}, - Size: sql.NullInt64{Int64: 100, Valid: true}, - FetchedAt: sql.NullTime{Time: fetched, Valid: true}, - } - if !accessed.IsZero() { - art.LastAccessedAt = sql.NullTime{Time: accessed, Valid: true} - } - if err := db.UpsertArtifact(art); err != nil { - t.Fatalf("UpsertArtifact: %v", err) - } -} - -func candidateNames(candidates []RetentionCandidate) map[string]RetentionCandidate { - byName := make(map[string]RetentionCandidate, len(candidates)) - for _, c := range candidates { - byName[c.Filename] = c - } - return byName -} - -func TestGetRetentionCandidates(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now() - cutoff := now.Add(-24 * time.Hour) - old := now.Add(-48 * time.Hour) - - seedRetentionArtifact(t, db, "npm", "stale", old, old) - seedRetentionArtifact(t, db, "npm", "never-served", old, time.Time{}) - seedRetentionArtifact(t, db, "npm", "recently-served", old, now) - // Fetched again after an eviction: the clear kept the old access time. - seedRetentionArtifact(t, db, "npm", "refetched", now, old) - seedRetentionArtifact(t, db, "gem", "other-ecosystem", old, old) - - maxID, err := db.MaxArtifactID() - if err != nil { - t.Fatalf("MaxArtifactID: %v", err) - } - candidates, err := db.GetRetentionCandidates(0, maxID, cutoff) - if err != nil { - t.Fatalf("GetRetentionCandidates: %v", err) - } - got := candidateNames(candidates) - - for _, want := range []string{"stale.tgz", "never-served.tgz", "other-ecosystem.tgz"} { - if _, ok := got[want]; !ok { - t.Errorf("%s missing from candidates", want) - } - } - for _, unwanted := range []string{"recently-served.tgz", "refetched.tgz"} { - if _, ok := got[unwanted]; ok { - t.Errorf("%s returned as a candidate", unwanted) - } - } - - stale := got["stale.tgz"] - if stale.Ecosystem != "npm" || stale.PackagePURL != "pkg:npm/stale" { - t.Errorf("stale candidate ecosystem/package = %q/%q, want npm/pkg:npm/stale", stale.Ecosystem, stale.PackagePURL) - } - if !stale.ExpiredBefore(cutoff) { - t.Error("stale candidate not expired before the cutoff") - } - if got["other-ecosystem.tgz"].Ecosystem != "gem" { - t.Errorf("other-ecosystem candidate ecosystem = %q, want gem", got["other-ecosystem.tgz"].Ecosystem) - } - }) -} - -func TestGetRetentionCandidatesWindow(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - old := time.Now().Add(-48 * time.Hour) - for _, name := range []string{"a", "b", "c"} { - seedRetentionArtifact(t, db, "npm", name, old, old) - } - cutoff := time.Now().Add(-time.Hour) - - first, err := db.GetRetentionCandidates(0, 1, cutoff) - if err != nil { - t.Fatalf("GetRetentionCandidates: %v", err) - } - rest, err := db.GetRetentionCandidates(1, 3, cutoff) - if err != nil { - t.Fatalf("GetRetentionCandidates: %v", err) - } - if len(first) != 1 || len(rest) != 2 { - t.Fatalf("window sizes = %d and %d, want 1 and 2", len(first), len(rest)) - } - if first[0].ID != 1 || rest[0].ID != 2 || rest[1].ID != 3 { - t.Errorf("window ids = %d, %d, %d, want 1, 2, 3", first[0].ID, rest[0].ID, rest[1].ID) - } - }) -} - -func TestClearExpiredArtifact(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now() - old := now.Add(-48 * time.Hour) - cutoff := now.Add(-24 * time.Hour) - seedRetentionArtifact(t, db, "npm", "stale", old, old) - seedRetentionArtifact(t, db, "npm", "served", old, old) - const stalePath = "npm/stale/1.0.0/stale.tgz" - - // A hit lands between the scan and the clear. - if err := db.RecordArtifactHit("pkg:npm/served@1.0.0", "served.tgz"); err != nil { - t.Fatalf("RecordArtifactHit: %v", err) - } - cleared, err := db.ClearExpiredArtifact("pkg:npm/served@1.0.0", "served.tgz", "npm/served/1.0.0/served.tgz", cutoff) - if err != nil || cleared { - t.Errorf("ClearExpiredArtifact(served) = %v, %v, want false, nil", cleared, err) - } - - cleared, err = db.ClearExpiredArtifact("pkg:npm/stale@1.0.0", "stale.tgz", "elsewhere", cutoff) - if err != nil || cleared { - t.Errorf("ClearExpiredArtifact(moved record) = %v, %v, want false, nil", cleared, err) - } - - cleared, err = db.ClearExpiredArtifact("pkg:npm/stale@1.0.0", "stale.tgz", stalePath, cutoff) - if err != nil || !cleared { - t.Fatalf("ClearExpiredArtifact(stale) = %v, %v, want true, nil", cleared, err) - } - art, err := db.GetArtifact("pkg:npm/stale@1.0.0", "stale.tgz") - if err != nil { - t.Fatalf("GetArtifact: %v", err) - } - if art.StoragePath.Valid || art.FetchedAt.Valid { - t.Error("stale artifact still cached after the clear") - } - if !art.LastAccessedAt.Valid { - t.Error("clear dropped last_accessed_at") - } - var queued int - if err := db.Get(&queued, db.Rebind(`SELECT COUNT(*) FROM pending_deletes WHERE path = ?`), stalePath); err != nil || queued != 1 { - t.Errorf("queued deletes for %s = %d, %v, want 1", stalePath, queued, err) - } - }) -} - -// When the object can't be queued for deletion, the clear is rolled back: -// the record keeps pointing at the object, so a later sweep retries it -// rather than leaving a file in storage that nothing refers to. -func TestClearExpiredArtifactRollsBackWhenQueueingFails(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now() - old := now.Add(-48 * time.Hour) - seedRetentionArtifact(t, db, "npm", "stale", old, old) - if _, err := db.Exec(`DROP TABLE pending_deletes`); err != nil { - t.Fatalf("dropping pending_deletes: %v", err) - } - - cleared, err := db.ClearExpiredArtifact("pkg:npm/stale@1.0.0", "stale.tgz", "npm/stale/1.0.0/stale.tgz", now.Add(-24*time.Hour)) - if err == nil || cleared { - t.Fatalf("ClearExpiredArtifact = %v, %v, want false and an error", cleared, err) - } - art, err := db.GetArtifact("pkg:npm/stale@1.0.0", "stale.tgz") - if err != nil { - t.Fatalf("GetArtifact: %v", err) - } - if !art.StoragePath.Valid || !art.FetchedAt.Valid { - t.Error("record cleared although its object couldn't be queued for deletion") - } - }) -} - -func TestFlushHitsWritesBufferedHits(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - old := time.Now().Add(-48 * time.Hour) - seedRetentionArtifact(t, db, "npm", "buffered", old, old) - db.BatchHits(time.Hour, nil) - - if err := db.RecordArtifactHit("pkg:npm/buffered@1.0.0", "buffered.tgz"); err != nil { - t.Fatalf("RecordArtifactHit: %v", err) - } - if err := db.FlushHits(); err != nil { - t.Fatalf("FlushHits: %v", err) - } - art, err := db.GetArtifact("pkg:npm/buffered@1.0.0", "buffered.tgz") - if err != nil { - t.Fatalf("GetArtifact: %v", err) - } - if !art.LastAccessedAt.Time.After(old) { - t.Errorf("last_accessed_at = %v after FlushHits, want the buffered hit", art.LastAccessedAt.Time) - } - }) -} - -func TestFlushHitsWithoutBatching(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - if err := db.FlushHits(); err != nil { - t.Errorf("FlushHits without batching = %v, want nil", err) - } - }) -} - -func TestMaxArtifactIDEmpty(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - maxID, err := db.MaxArtifactID() - if err != nil || maxID != 0 { - t.Errorf("MaxArtifactID() = %d, %v, want 0, nil", maxID, err) - } - }) -} - -func TestGetArtifactEcosystem(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - seedRetentionArtifact(t, db, "rubygems", "rails", time.Now(), time.Time{}) - - got, err := db.GetArtifactEcosystem("pkg:rubygems/rails@1.0.0") - if err != nil || got != "rubygems" { - t.Errorf("GetArtifactEcosystem() = %q, %v, want rubygems, nil", got, err) - } - got, err = db.GetArtifactEcosystem("pkg:npm/missing@1.0.0") - if err != nil || got != "" { - t.Errorf("GetArtifactEcosystem(missing) = %q, %v, want empty, nil", got, err) - } - }) -} - -func TestPendingDeletesQueuedAtIndex(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - indexQuery := `SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_pending_deletes_queued_at'` - if db.Dialect() == DialectPostgres { - indexQuery = `SELECT COUNT(*) FROM pg_indexes WHERE indexname = 'idx_pending_deletes_queued_at'` - } - hasIndex := func() bool { - t.Helper() - var count int - if err := db.Get(&count, indexQuery); err != nil { - t.Fatalf("querying indexes: %v", err) - } - return count == 1 - } - - if !hasIndex() { - t.Error("fresh schema has no idx_pending_deletes_queued_at") - } - if _, err := db.Exec(`DROP INDEX idx_pending_deletes_queued_at`); err != nil { - t.Fatalf("dropping index: %v", err) - } - if err := migrateAddPendingDeletesQueuedAtIndex(db); err != nil { - t.Fatalf("migration: %v", err) - } - if err := migrateAddPendingDeletesQueuedAtIndex(db); err != nil { - t.Fatalf("migration is not idempotent: %v", err) - } - if !hasIndex() { - t.Error("migration did not create idx_pending_deletes_queued_at") - } - }) -} - -func TestRetentionCandidateExpiredBefore(t *testing.T) { - now := time.Now() - cutoff := now.Add(-24 * time.Hour) - old := sql.NullTime{Time: now.Add(-48 * time.Hour), Valid: true} - recent := sql.NullTime{Time: now, Valid: true} - - tests := []struct { - name string - fetched, accessed, mirrored sql.NullTime - want bool - }{ - {"fetched and accessed before", old, old, sql.NullTime{}, true}, - {"never served", old, sql.NullTime{}, sql.NullTime{}, true}, - {"accessed since", old, recent, sql.NullTime{}, false}, - {"fetched again since", recent, old, sql.NullTime{}, false}, - {"not cached", sql.NullTime{}, old, sql.NullTime{}, false}, - {"mirrored since", old, old, recent, false}, - {"mirrored before", old, old, old, true}, - } - for _, tt := range tests { - c := RetentionCandidate{FetchedAt: tt.fetched, LastAccessedAt: tt.accessed, MirroredAt: tt.mirrored} - if got := c.ExpiredBefore(cutoff); got != tt.want { - t.Errorf("%s: ExpiredBefore() = %v, want %v", tt.name, got, tt.want) - } - } -} - -// A TIMESTAMP column on Postgres holds the local wall clock time without a -// zone. Read back, the candidate's times must still compare correctly -// against a cutoff east of UTC, where a time labelled UTC would look hours -// newer than it is. -func TestGetRetentionCandidatesTimezone(t *testing.T) { - tokyo, err := time.LoadLocation("Asia/Tokyo") - if err != nil { - t.Skipf("time zone data unavailable: %v", err) - } - prev := time.Local - time.Local = tokyo - t.Cleanup(func() { time.Local = prev }) - - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.Now() - fetched := now.Add(-90 * time.Minute) - seedRetentionArtifact(t, db, "npm", "tz", fetched, time.Time{}) - cutoff := now.Add(-60 * time.Minute) - - candidates, err := db.GetRetentionCandidates(0, 10, cutoff) - if err != nil { - t.Fatalf("GetRetentionCandidates: %v", err) - } - if len(candidates) != 1 { - t.Fatalf("got %d candidates, want 1", len(candidates)) - } - c := candidates[0] - if !c.ExpiredBefore(cutoff) { - t.Errorf("candidate fetched at %v not expired before %v", c.FetchedAt.Time, cutoff) - } - if diff := c.FetchedAt.Time.Sub(fetched).Abs(); diff > time.Second { - t.Errorf("fetched_at read back as %v, want %v", c.FetchedAt.Time, fetched) - } - }) -} - -func TestArtifactFilenameIndex(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - indexQuery := `SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_artifacts_filename'` - if db.Dialect() == DialectPostgres { - indexQuery = `SELECT COUNT(*) FROM pg_indexes WHERE indexname = 'idx_artifacts_filename'` - } - hasIndex := func() bool { - t.Helper() - var count int - if err := db.Get(&count, indexQuery); err != nil { - t.Fatalf("querying indexes: %v", err) - } - return count == 1 - } - - if !hasIndex() { - t.Error("fresh schema has no idx_artifacts_filename") - } - if _, err := db.Exec(`DROP INDEX idx_artifacts_filename`); err != nil { - t.Fatalf("dropping index: %v", err) - } - if err := migrateAddArtifactFilenameIndex(db); err != nil { - t.Fatalf("migration: %v", err) - } - if err := migrateAddArtifactFilenameIndex(db); err != nil { - t.Fatalf("migration is not idempotent: %v", err) - } - if !hasIndex() { - t.Error("migration did not create idx_artifacts_filename") - } - }) -} diff --git a/internal/database/schema.go b/internal/database/schema.go index 6a9adaa..c8d8d1e 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -1,7 +1,6 @@ package database import ( - "context" "fmt" "strings" "time" @@ -73,35 +72,12 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count INTEGER DEFAULT 0, last_accessed_at DATETIME, created_at DATETIME, - updated_at DATETIME, - source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT, - mirrored_at DATETIME + updated_at DATETIME ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); -CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); -CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); -CREATE TABLE IF NOT EXISTS pending_deletes ( - path TEXT NOT NULL PRIMARY KEY, - queued_at DATETIME NOT NULL -); -CREATE INDEX IF NOT EXISTS idx_pending_deletes_queued_at ON pending_deletes(queued_at); - -CREATE TABLE IF NOT EXISTS job_runs ( - name TEXT NOT NULL PRIMARY KEY, - started_at DATETIME NOT NULL -); - -CREATE TABLE IF NOT EXISTS fetch_leases ( - key TEXT NOT NULL PRIMARY KEY, - owner TEXT NOT NULL, - expires_at DATETIME NOT NULL, - failed BOOLEAN NOT NULL DEFAULT FALSE -); - CREATE TABLE IF NOT EXISTS vulnerabilities ( id INTEGER PRIMARY KEY, vuln_id TEXT NOT NULL, @@ -125,69 +101,14 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( name TEXT NOT NULL, storage_path TEXT NOT NULL, etag TEXT, - link TEXT, content_type TEXT, - content_encoding TEXT, - content_digest TEXT, size INTEGER, last_modified DATETIME, fetched_at DATETIME, - last_accessed_at DATETIME, created_at DATETIME, updated_at DATETIME ); CREATE UNIQUE INDEX IF NOT EXISTS idx_metadata_eco_name ON metadata_cache(ecosystem, name); -CREATE INDEX IF NOT EXISTS idx_metadata_storage_path ON metadata_cache(storage_path); -CREATE INDEX IF NOT EXISTS idx_metadata_last_used ON metadata_cache(COALESCE(last_accessed_at, updated_at)); -CREATE INDEX IF NOT EXISTS idx_artifacts_last_used ON artifacts(COALESCE(last_accessed_at, fetched_at)); - -CREATE TABLE IF NOT EXISTS metadata_dates ( - ecosystem TEXT NOT NULL, - proxy_url TEXT NOT NULL, - name TEXT NOT NULL, - etag TEXT NOT NULL, - last_modified INTEGER NOT NULL, - PRIMARY KEY (ecosystem, proxy_url, name) -); - -CREATE TABLE IF NOT EXISTS upstream_holds ( - host TEXT NOT NULL PRIMARY KEY, - until_ms INTEGER NOT NULL -); - -CREATE TABLE IF NOT EXISTS metadata_snapshots ( - id INTEGER PRIMARY KEY, - ecosystem TEXT NOT NULL, - name TEXT NOT NULL, - storage_path TEXT NOT NULL, - content_type TEXT, - content_encoding TEXT, - content_digest TEXT NOT NULL, - size INTEGER NOT NULL, - valid_from_ms INTEGER NOT NULL -); -CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms); -CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path); - -CREATE TABLE IF NOT EXISTS projects ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - ref TEXT, - branch TEXT, - uploaded_at DATETIME NOT NULL -); -CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name); -CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at); - -CREATE TABLE IF NOT EXISTS project_components ( - project_id INTEGER NOT NULL, - package_purl TEXT NOT NULL, - ecosystem TEXT NOT NULL, - name TEXT NOT NULL, - version TEXT NOT NULL, - PRIMARY KEY (project_id, package_purl, version) -); -CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version); CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, @@ -251,35 +172,12 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count BIGINT DEFAULT 0, last_accessed_at TIMESTAMP, created_at TIMESTAMP, - updated_at TIMESTAMP, - source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT, - mirrored_at TIMESTAMP + updated_at TIMESTAMP ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); -CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); -CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); -CREATE TABLE IF NOT EXISTS pending_deletes ( - path TEXT NOT NULL PRIMARY KEY, - queued_at TIMESTAMP NOT NULL -); -CREATE INDEX IF NOT EXISTS idx_pending_deletes_queued_at ON pending_deletes(queued_at); - -CREATE TABLE IF NOT EXISTS job_runs ( - name TEXT NOT NULL PRIMARY KEY, - started_at TIMESTAMP NOT NULL -); - -CREATE TABLE IF NOT EXISTS fetch_leases ( - key TEXT NOT NULL PRIMARY KEY, - owner TEXT NOT NULL, - expires_at TIMESTAMP NOT NULL, - failed BOOLEAN NOT NULL DEFAULT FALSE -); - CREATE TABLE IF NOT EXISTS vulnerabilities ( id SERIAL PRIMARY KEY, vuln_id TEXT NOT NULL, @@ -303,69 +201,14 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( name TEXT NOT NULL, storage_path TEXT NOT NULL, etag TEXT, - link TEXT, content_type TEXT, - content_encoding TEXT, - content_digest TEXT, size BIGINT, last_modified TIMESTAMP, fetched_at TIMESTAMP, - last_accessed_at TIMESTAMP, created_at TIMESTAMP, updated_at TIMESTAMP ); CREATE UNIQUE INDEX IF NOT EXISTS idx_metadata_eco_name ON metadata_cache(ecosystem, name); -CREATE INDEX IF NOT EXISTS idx_metadata_storage_path ON metadata_cache(storage_path); -CREATE INDEX IF NOT EXISTS idx_metadata_last_used ON metadata_cache((COALESCE(last_accessed_at, updated_at)) NULLS FIRST); -CREATE INDEX IF NOT EXISTS idx_artifacts_last_used ON artifacts((COALESCE(last_accessed_at, fetched_at)) NULLS FIRST); - -CREATE TABLE IF NOT EXISTS metadata_dates ( - ecosystem TEXT NOT NULL, - proxy_url TEXT NOT NULL, - name TEXT NOT NULL, - etag TEXT NOT NULL, - last_modified BIGINT NOT NULL, - PRIMARY KEY (ecosystem, proxy_url, name) -); - -CREATE TABLE IF NOT EXISTS upstream_holds ( - host TEXT NOT NULL PRIMARY KEY, - until_ms BIGINT NOT NULL -); - -CREATE TABLE IF NOT EXISTS metadata_snapshots ( - id SERIAL PRIMARY KEY, - ecosystem TEXT NOT NULL, - name TEXT NOT NULL, - storage_path TEXT NOT NULL, - content_type TEXT, - content_encoding TEXT, - content_digest TEXT NOT NULL, - size BIGINT NOT NULL, - valid_from_ms BIGINT NOT NULL -); -CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms); -CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path); - -CREATE TABLE IF NOT EXISTS projects ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - ref TEXT, - branch TEXT, - uploaded_at TIMESTAMP NOT NULL -); -CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name); -CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at); - -CREATE TABLE IF NOT EXISTS project_components ( - project_id INTEGER NOT NULL, - package_purl TEXT NOT NULL, - ecosystem TEXT NOT NULL, - name TEXT NOT NULL, - version TEXT NOT NULL, - PRIMARY KEY (project_id, package_purl, version) -); -CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version); CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, @@ -388,15 +231,10 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count INTEGER DEFAULT 0, last_accessed_at DATETIME, created_at DATETIME, - updated_at DATETIME, - source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT, - mirrored_at DATETIME + updated_at DATETIME ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); -CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); -CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); ` @@ -414,15 +252,10 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count BIGINT DEFAULT 0, last_accessed_at TIMESTAMP, created_at TIMESTAMP, - updated_at TIMESTAMP, - source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT, - mirrored_at TIMESTAMP + updated_at TIMESTAMP ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); -CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); -CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); ` @@ -526,23 +359,6 @@ var migrations = []migration{ {"003_ensure_artifacts_table", migrateEnsureArtifactsTable}, {"004_ensure_vulnerabilities_table", migrateEnsureVulnerabilitiesTable}, {"005_ensure_metadata_cache_table", migrateEnsureMetadataCacheTable}, - {"006_add_metadata_content_digest", migrateAddMetadataContentDigest}, - {"007_add_metadata_link", migrateAddMetadataLink}, - {"008_add_metadata_content_encoding", migrateAddMetadataContentEncoding}, - {"009_add_pending_deletes", migrateAddPendingDeletes}, - {"010_add_pending_deletes_queued_at_index", migrateAddPendingDeletesQueuedAtIndex}, - {"011_add_metadata_dates", migrateAddMetadataDates}, - {"012_add_artifact_source", migrateAddArtifactSource}, - {"013_add_artifact_manifest", migrateAddArtifactManifest}, - {"014_add_metadata_last_accessed", migrateAddMetadataLastAccessed}, - {"015_add_job_runs", migrateAddJobRuns}, - {"016_add_fetch_leases", migrateAddFetchLeases}, - {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, - {"018_add_upstream_holds", migrateAddUpstreamHolds}, - {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, - {"020_add_metadata_snapshots", migrateAddMetadataSnapshots}, - {"021_add_projects", migrateAddProjects}, - {"022_add_artifact_mirrored_at", migrateAddArtifactMirroredAt}, } // isTableNotFound returns true if the error indicates a missing table. @@ -612,19 +428,9 @@ func (db *DB) recordAllMigrations() error { return nil } -// schemaLock serialises schema creation and migrations across instances -// sharing a Postgres database. -const schemaLock = "schema" - // MigrateSchema applies any unapplied migrations in order. // For a fully migrated database this executes a single SELECT query. -// Instances sharing a Postgres database run it one after the other, so each -// sees the migrations the one before it recorded. func (db *DB) MigrateSchema() error { - return db.WithLock(context.Background(), schemaLock, db.migrateSchema) -} - -func (db *DB) migrateSchema() error { applied, err := db.appliedMigrations() if err != nil { return err @@ -775,242 +581,6 @@ func migrateEnsureMetadataCacheTable(db *DB) error { return db.EnsureMetadataCacheTable() } -func migrateAddMetadataContentDigest(db *DB) error { - hasColumn, err := db.HasColumn("metadata_cache", "content_digest") - if err != nil { - return fmt.Errorf("checking metadata_cache content_digest column: %w", err) - } - if hasColumn { - return nil - } - if _, err := db.Exec("ALTER TABLE metadata_cache ADD COLUMN content_digest TEXT"); err != nil { - return fmt.Errorf("adding metadata_cache content_digest column: %w", err) - } - return nil -} - -func migrateAddMetadataLink(db *DB) error { - hasColumn, err := db.HasColumn("metadata_cache", "link") - if err != nil { - return fmt.Errorf("checking metadata_cache link column: %w", err) - } - if hasColumn { - return nil - } - if _, err := db.Exec("ALTER TABLE metadata_cache ADD COLUMN link TEXT"); err != nil { - return fmt.Errorf("adding metadata_cache link column: %w", err) - } - return nil -} - -func migrateAddMetadataContentEncoding(db *DB) error { - hasColumn, err := db.HasColumn("metadata_cache", "content_encoding") - if err != nil { - return fmt.Errorf("checking metadata_cache content_encoding column: %w", err) - } - if hasColumn { - return nil - } - if _, err := db.Exec("ALTER TABLE metadata_cache ADD COLUMN content_encoding TEXT"); err != nil { - return fmt.Errorf("adding metadata_cache content_encoding column: %w", err) - } - // Rows cached before this column existed hold transport-decompressed bytes - // with no recorded encoding and the upstream ETag captured under Go's - // auto-added Accept-Encoding: gzip. Clearing the validators forces one fresh - // fetch per key so the encoding is recorded and verbatim bytes are restored, - // instead of an If-None-Match 304 re-serving the stale decompressed copy. - if _, err := db.Exec("UPDATE metadata_cache SET etag = NULL, fetched_at = NULL"); err != nil { - return fmt.Errorf("invalidating metadata_cache validators: %w", err) - } - return nil -} - -// migrateAddPendingDeletes creates the queue of storage paths that no record -// points at any more, which the server deletes after a grace period. -func migrateAddPendingDeletes(db *DB) error { - ts := sqliteDatetime - if db.dialect == DialectPostgres { - ts = postgresTimestamp - } - query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS pending_deletes ( - path TEXT NOT NULL PRIMARY KEY, - queued_at %s NOT NULL - )`, ts) - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("creating pending_deletes table: %w", err) - } - return nil -} - -// migrateAddJobRuns creates the table that records when a scheduled job -// last started, so instances sharing the database run it once per interval. -func migrateAddJobRuns(db *DB) error { - ts := sqliteDatetime - if db.dialect == DialectPostgres { - ts = postgresTimestamp - } - query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS job_runs ( - name TEXT NOT NULL PRIMARY KEY, - started_at %s NOT NULL - )`, ts) - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("creating job_runs table: %w", err) - } - return nil -} - -// migrateAddFetchLeases creates the table instances sharing a Postgres -// database use to fetch each cache miss from upstream only once. -func migrateAddFetchLeases(db *DB) error { - ts := sqliteDatetime - if db.dialect == DialectPostgres { - ts = postgresTimestamp - } - query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS fetch_leases ( - key TEXT NOT NULL PRIMARY KEY, - owner TEXT NOT NULL, - expires_at %s NOT NULL, - failed BOOLEAN NOT NULL DEFAULT FALSE - )`, ts) - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("creating fetch_leases table: %w", err) - } - return nil -} - -// migrateAddUpstreamHolds creates the table through which instances sharing -// the database honour each other's upstream rate-limit holds. -func migrateAddUpstreamHolds(db *DB) error { - millis := "INTEGER" - if db.dialect == DialectPostgres { - millis = "BIGINT" - } - query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS upstream_holds ( - host TEXT NOT NULL PRIMARY KEY, - until_ms %s NOT NULL - )`, millis) - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("creating upstream_holds table: %w", err) - } - return nil -} - -// migrateAddProjects creates the tables of the project inventory: the -// projects that uploaded an SBOM and the package versions each one uses. -func migrateAddProjects(db *DB) error { - id, ts := "INTEGER PRIMARY KEY", sqliteDatetime - if db.dialect == DialectPostgres { - id, ts = "SERIAL PRIMARY KEY", postgresTimestamp - } - statements := []string{ - fmt.Sprintf(`CREATE TABLE IF NOT EXISTS projects ( - id %s, - name TEXT NOT NULL, - ref TEXT, - branch TEXT, - uploaded_at %s NOT NULL - )`, id, ts), - `CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name)`, - `CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at)`, - `CREATE TABLE IF NOT EXISTS project_components ( - project_id INTEGER NOT NULL, - package_purl TEXT NOT NULL, - ecosystem TEXT NOT NULL, - name TEXT NOT NULL, - version TEXT NOT NULL, - PRIMARY KEY (project_id, package_purl, version) - )`, - `CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version)`, - } - for _, stmt := range statements { - if _, err := db.Exec(stmt); err != nil { - return fmt.Errorf("creating project tables: %w", err) - } - } - return nil -} - -// migrateAddPendingDeletesQueuedAtIndex indexes the column reclaim orders -// the queue by, which grows large after a retention sweep. -func migrateAddPendingDeletesQueuedAtIndex(db *DB) error { - _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_pending_deletes_queued_at ON pending_deletes(queued_at)`) - if err != nil { - return fmt.Errorf("creating pending_deletes queued_at index: %w", err) - } - return nil -} - -// migrateAddArtifactContentHashIndex indexes artifacts by content, so an -// artifact already stored under another name can be found by its hash. -func migrateAddArtifactContentHashIndex(db *DB) error { - _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash)`) - if err != nil { - return fmt.Errorf("creating artifacts content_hash index: %w", err) - } - return nil -} - -// migrateAddArtifactFilenameIndex indexes artifacts by filename, for -// lookups that don't know the version, such as CachedArtifactNames. -func migrateAddArtifactFilenameIndex(db *DB) error { - _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename)`) - if err != nil { - return fmt.Errorf("creating artifacts filename index: %w", err) - } - return nil -} - -// migrateAddMetadataSnapshots creates the table of kept earlier versions of -// metadata documents, which dated snapshot requests are served from. -func migrateAddMetadataSnapshots(db *DB) error { - pk, integer := "id INTEGER PRIMARY KEY", "INTEGER" - if db.dialect == DialectPostgres { - pk, integer = "id SERIAL PRIMARY KEY", "BIGINT" - } - statements := []string{ - fmt.Sprintf(`CREATE TABLE IF NOT EXISTS metadata_snapshots ( - %s, - ecosystem TEXT NOT NULL, - name TEXT NOT NULL, - storage_path TEXT NOT NULL, - content_type TEXT, - content_encoding TEXT, - content_digest TEXT NOT NULL, - size %s NOT NULL, - valid_from_ms %s NOT NULL - )`, pk, integer, integer), - `CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms)`, - `CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path)`, - } - for _, query := range statements { - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("creating metadata_snapshots: %w", err) - } - } - return nil -} - -// migrateAddMetadataDates creates the table of Last-Modified dates for -// rewritten metadata, which replicas sharing the database agree on. -func migrateAddMetadataDates(db *DB) error { - seconds := "INTEGER" - if db.dialect == DialectPostgres { - seconds = "BIGINT" - } - query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS metadata_dates ( - ecosystem TEXT NOT NULL, - proxy_url TEXT NOT NULL, - name TEXT NOT NULL, - etag TEXT NOT NULL, - last_modified %s NOT NULL, - PRIMARY KEY (ecosystem, proxy_url, name) - )`, seconds) - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("creating metadata_dates table: %w", err) - } - return nil -} - // EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist. func (db *DB) EnsureMetadataCacheTable() error { has, err := db.HasTable("metadata_cache") @@ -1029,11 +599,8 @@ func (db *DB) EnsureMetadataCacheTable() error { ecosystem TEXT NOT NULL, name TEXT NOT NULL, storage_path TEXT NOT NULL, - etag TEXT, - link TEXT, - content_type TEXT, - content_encoding TEXT, - content_digest TEXT, + etag TEXT, + content_type TEXT, size BIGINT, last_modified TIMESTAMP, fetched_at TIMESTAMP, @@ -1049,11 +616,8 @@ func (db *DB) EnsureMetadataCacheTable() error { ecosystem TEXT NOT NULL, name TEXT NOT NULL, storage_path TEXT NOT NULL, - etag TEXT, - link TEXT, - content_type TEXT, - content_encoding TEXT, - content_digest TEXT, + etag TEXT, + content_type TEXT, size INTEGER, last_modified DATETIME, fetched_at DATETIME, @@ -1068,89 +632,3 @@ func (db *DB) EnsureMetadataCacheTable() error { } return nil } - -// migrateAddArtifactSource records whether an artifact is a cached copy of an -// upstream registry or was pushed to the proxy. Existing rows are all upstream. -func migrateAddArtifactSource(db *DB) error { - hasCol, err := db.HasColumn("artifacts", "source") - if err != nil { - return fmt.Errorf("checking column source: %w", err) - } - if hasCol { - return nil - } - if _, err := db.Exec(`ALTER TABLE artifacts ADD COLUMN source TEXT NOT NULL DEFAULT 'upstream'`); err != nil { - return fmt.Errorf("adding column source to artifacts: %w", err) - } - return nil -} - -// migrateAddArtifactMirroredAt records when a mirror run last found an -// artifact cached, which retention counts as use and LRU eviction doesn't. -func migrateAddArtifactMirroredAt(db *DB) error { - hasCol, err := db.HasColumn("artifacts", "mirrored_at") - if err != nil { - return fmt.Errorf("checking column mirrored_at: %w", err) - } - if hasCol { - return nil - } - ts := sqliteDatetime - if db.dialect == DialectPostgres { - ts = postgresTimestamp - } - if _, err := db.Exec(fmt.Sprintf("ALTER TABLE artifacts ADD COLUMN mirrored_at %s", ts)); err != nil { - return fmt.Errorf("adding column mirrored_at to artifacts: %w", err) - } - return nil -} - -// migrateAddArtifactManifest stores the manifest read from a pushed archive, -// which the metadata of its ecosystem lists the version with. -func migrateAddArtifactManifest(db *DB) error { - hasCol, err := db.HasColumn("artifacts", "manifest") - if err != nil { - return fmt.Errorf("checking column manifest: %w", err) - } - if hasCol { - return nil - } - if _, err := db.Exec(`ALTER TABLE artifacts ADD COLUMN manifest TEXT`); err != nil { - return fmt.Errorf("adding column manifest to artifacts: %w", err) - } - return nil -} - -// migrateAddMetadataLastAccessed records when cached metadata was last served, -// for size eviction. Existing rows are backfilled in the background -// (BackfillMetadataLastAccessed), so startup does not wait for them. -func migrateAddMetadataLastAccessed(db *DB) error { - hasCol, err := db.HasColumn("metadata_cache", "last_accessed_at") - if err != nil { - return fmt.Errorf("checking column last_accessed_at: %w", err) - } - if !hasCol { - ts := sqliteDatetime - if db.dialect == DialectPostgres { - ts = postgresTimestamp - } - if _, err := db.Exec(fmt.Sprintf("ALTER TABLE metadata_cache ADD COLUMN last_accessed_at %s", ts)); err != nil { - return fmt.Errorf("adding column last_accessed_at to metadata_cache: %w", err) - } - } - indexes := []string{ - `CREATE INDEX IF NOT EXISTS idx_metadata_storage_path ON metadata_cache(storage_path)`, - `CREATE INDEX IF NOT EXISTS idx_metadata_last_used ON metadata_cache(COALESCE(last_accessed_at, updated_at))`, - `CREATE INDEX IF NOT EXISTS idx_artifacts_last_used ON artifacts(COALESCE(last_accessed_at, fetched_at))`, - } - if db.dialect == DialectPostgres { - indexes[1] = `CREATE INDEX IF NOT EXISTS idx_metadata_last_used ON metadata_cache((COALESCE(last_accessed_at, updated_at)) NULLS FIRST)` - indexes[2] = `CREATE INDEX IF NOT EXISTS idx_artifacts_last_used ON artifacts((COALESCE(last_accessed_at, fetched_at)) NULLS FIRST)` - } - for _, q := range indexes { - if _, err := db.Exec(q); err != nil { - return fmt.Errorf("indexing metadata_cache: %w", err) - } - } - return nil -} diff --git a/internal/database/types.go b/internal/database/types.go index c2675d1..47dc47e 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -2,11 +2,8 @@ package database import ( "database/sql" - "net/url" "strings" "time" - - "github.com/git-pkgs/artifacts" ) // Package represents a package in the database. @@ -50,89 +47,12 @@ type Version struct { // Version extracts the version string from the PURL. // e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21" func (v *Version) Version() string { - return VersionFromPURL(v.PURL) + if idx := strings.LastIndex(v.PURL, "@"); idx >= 0 { + return v.PURL[idx+1:] + } + return "" } -// EscapedVersion returns the version escaped for use as a single URL path -// segment. -// -// Version returns decoded text, which is what should be shown to a user but is -// not safe to drop into a link: html/template preserves reserved characters and -// existing escapes in a URL, so "release/1" would split into two path segments, -// "v1?build" would start a query string, and a literal "%2B" would be read back -// as "+". Escaping here and decoding in splitWildcardPath round-trips the value, -// so the link resolves to the version that was stored. -func (v *Version) EscapedVersion() string { - return url.PathEscape(v.Version()) -} - -// DisplayPURL returns the PURL with its path components percent-decoded, for -// showing in the UI. The stored PURL keeps the canonical encoding (which is -// what the API and all lookups use); this is only a readable rendering, so that -// a version like "7.91+dfsg1-2ubuntu0.1" is not shown as "7.91%2Bdfsg1-2ubuntu0.1" -// and an npm scope is shown as "@babel" rather than "%40babel". Qualifiers and -// subpath keep their encoding, since decoding those would be ambiguous. -func (v *Version) DisplayPURL() string { - base, suffix := v.PURL, "" - if i := strings.IndexAny(base, "?#"); i >= 0 { - base, suffix = base[:i], base[i:] - } - - name, version := base, "" - if idx := strings.LastIndex(base, "@"); idx >= 0 { - name, version = base[:idx], "@"+decodePURLComponent(base[idx+1:]) - } - - parts := strings.Split(name, "/") - for i, part := range parts { - parts[i] = decodePURLComponent(part) - } - return strings.Join(parts, "/") + version + suffix -} - -// VersionFromPURL extracts the decoded version string from a PURL. -// -// PURL percent-encodes characters that are not safe in a path component, so a -// Debian version like "7.91+dfsg1-2ubuntu0.1" is stored as -// "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1". The raw substring after "@" is -// therefore not the version: it must be percent-decoded before being displayed -// or used to build a URL, otherwise "%2B" leaks into the UI and round-tripping -// the value back into a PURL double-encodes it. -// -// e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21" -func VersionFromPURL(p string) string { - // Qualifiers ("?key=value") and subpath ("#path") follow the version. - if i := strings.IndexAny(p, "?#"); i >= 0 { - p = p[:i] - } - idx := strings.LastIndex(p, "@") - if idx < 0 { - return "" - } - return decodePURLComponent(p[idx+1:]) -} - -// decodePURLComponent percent-decodes a single PURL path component, returning -// the input unchanged if it is not valid percent-encoding. -func decodePURLComponent(s string) string { - if !strings.Contains(s, "%") { - return s - } - decoded, err := url.PathUnescape(s) - if err != nil { - return s - } - return decoded -} - -// Artifact sources. Upstream artifacts are a cache of a registry and may be -// evicted; local artifacts were pushed to the proxy and are only removed by a -// delete. -const ( - SourceUpstream = "upstream" - SourceLocal = "local" -) - // Artifact represents a cached artifact in the database. // This table is proxy-specific and not part of git-pkgs. type Artifact struct { @@ -149,18 +69,6 @@ type Artifact struct { LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"` CreatedAt time.Time `db:"created_at" json:"created_at"` UpdatedAt time.Time `db:"updated_at" json:"updated_at"` - Source string `db:"source" json:"source"` - Manifest sql.NullString `db:"manifest" json:"manifest,omitempty"` - // MirroredAt is when a mirror run last found the artifact cached. It - // keeps the artifact from retention but not from LRU eviction. - MirroredAt sql.NullTime `db:"mirrored_at" json:"-"` -} - -func (a *Artifact) sourceOrUpstream() string { - if a.Source == "" { - return SourceUpstream - } - return a.Source } // IsCached returns true if the artifact has been fetched and stored locally. @@ -168,31 +76,19 @@ func (a *Artifact) IsCached() bool { return a.StoragePath.Valid && a.FetchedAt.Valid } -// CachedArtifact contains the fields needed to serve a cached artifact. -type CachedArtifact struct { - Ecosystem string - StoragePath string - Artifact artifacts.Artifact - Integrity sql.NullString -} - // MetadataCacheEntry represents a cached metadata blob for offline serving. type MetadataCacheEntry struct { - ID int64 `db:"id" json:"id"` - Ecosystem string `db:"ecosystem" json:"ecosystem"` - Name string `db:"name" json:"name"` - StoragePath string `db:"storage_path" json:"storage_path"` - ETag sql.NullString `db:"etag" json:"etag,omitempty"` - Link sql.NullString `db:"link" json:"link,omitempty"` - ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"` - ContentEncoding sql.NullString `db:"content_encoding" json:"content_encoding,omitempty"` - ContentDigest sql.NullString `db:"content_digest" json:"content_digest,omitempty"` - Size sql.NullInt64 `db:"size" json:"size,omitempty"` - LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` - FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` - LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + ID int64 `db:"id" json:"id"` + Ecosystem string `db:"ecosystem" json:"ecosystem"` + Name string `db:"name" json:"name"` + StoragePath string `db:"storage_path" json:"storage_path"` + ETag sql.NullString `db:"etag" json:"etag,omitempty"` + ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"` + Size sql.NullInt64 `db:"size" json:"size,omitempty"` + LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` + FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } // Vulnerability represents a cached vulnerability record. diff --git a/internal/database/upstream_holds.go b/internal/database/upstream_holds.go deleted file mode 100644 index 6f842ca..0000000 --- a/internal/database/upstream_holds.go +++ /dev/null @@ -1,58 +0,0 @@ -package database - -import ( - "fmt" - "time" -) - -// Upstream holds let instances sharing a database honour a rate-limit hold -// any one of them placed, so each does not need its own 429 before it stops -// sending. Times are the instances' own clocks in Unix milliseconds; a few -// seconds of skew does not matter next to holds of a minute or more. - -type upstreamHoldRow struct { - Host string `db:"host"` - UntilMS int64 `db:"until_ms"` -} - -// PutUpstreamHold records that requests to host are held until until. A hold -// already recorded for longer is kept. -func (db *DB) PutUpstreamHold(host string, until time.Time) error { - _, err := db.Exec(db.Rebind(` - INSERT INTO upstream_holds (host, until_ms) VALUES (?, ?) - ON CONFLICT (host) DO UPDATE SET until_ms = CASE - WHEN excluded.until_ms > upstream_holds.until_ms THEN excluded.until_ms - ELSE upstream_holds.until_ms END - `), host, until.UnixMilli()) - if err != nil { - return fmt.Errorf("recording upstream hold: %w", err) - } - return nil -} - -// UpstreamHolds returns the holds that have not ended by now, and deletes -// those that have. -func (db *DB) UpstreamHolds(now time.Time) (map[string]time.Time, error) { - var rows []upstreamHoldRow - if err := db.Select(&rows, `SELECT host, until_ms FROM upstream_holds`); err != nil { - return nil, fmt.Errorf("reading upstream holds: %w", err) - } - holds := make(map[string]time.Time, len(rows)) - expired := false - for _, row := range rows { - until := time.UnixMilli(row.UntilMS) - if now.Before(until) { - holds[row.Host] = until - } else { - expired = true - } - } - // Only write when there is something to delete, so the frequent reads - // stay reads. - if expired { - if _, err := db.Exec(db.Rebind(`DELETE FROM upstream_holds WHERE until_ms <= ?`), now.UnixMilli()); err != nil { - return holds, fmt.Errorf("deleting expired upstream holds: %w", err) - } - } - return holds, nil -} diff --git a/internal/database/upstream_holds_test.go b/internal/database/upstream_holds_test.go deleted file mode 100644 index 5d6617c..0000000 --- a/internal/database/upstream_holds_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package database - -import ( - "testing" - "time" -) - -func TestUpstreamHolds(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - now := time.UnixMilli(time.Now().UnixMilli()) - if err := db.PutUpstreamHold("registry.npmjs.org", now.Add(time.Minute)); err != nil { - t.Fatal(err) - } - // A shorter hold reported later keeps the longer one. - if err := db.PutUpstreamHold("registry.npmjs.org", now.Add(30*time.Second)); err != nil { - t.Fatal(err) - } - if err := db.PutUpstreamHold("pypi.org", now.Add(10*time.Second)); err != nil { - t.Fatal(err) - } - holds, err := db.UpstreamHolds(now) - if err != nil { - t.Fatal(err) - } - if len(holds) != 2 || !holds["registry.npmjs.org"].Equal(now.Add(time.Minute)) { - t.Fatalf("holds = %v", holds) - } - - // A longer one replaces it. - if err := db.PutUpstreamHold("registry.npmjs.org", now.Add(time.Hour)); err != nil { - t.Fatal(err) - } - holds, err = db.UpstreamHolds(now.Add(20 * time.Second)) - if err != nil { - t.Fatal(err) - } - if len(holds) != 1 || !holds["registry.npmjs.org"].Equal(now.Add(time.Hour)) { - t.Fatalf("holds after pypi.org ended = %v", holds) - } - var rows int - if err := db.Get(&rows, "SELECT COUNT(*) FROM upstream_holds"); err != nil { - t.Fatal(err) - } - if rows != 1 { - t.Errorf("rows = %d, want the ended hold deleted", rows) - } - }) -} diff --git a/internal/database/version_purl_test.go b/internal/database/version_purl_test.go deleted file mode 100644 index 517022b..0000000 --- a/internal/database/version_purl_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package database - -import ( - "database/sql" - "net/url" - "testing" - "time" -) - -func TestVersionFromPURL(t *testing.T) { - tests := []struct { - name string - purl string - want string - }{ - {"simple", "pkg:npm/lodash@4.17.21", "4.17.21"}, - {"namespaced", "pkg:composer/symfony/console@6.0.0", "6.0.0"}, - // Debian/Ubuntu versions routinely contain "+", which PURL encodes. - {"encoded plus", "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"}, - {"encoded epoch", "pkg:deb/curl@1%3A7.81.0-1", "1:7.81.0-1"}, - {"encoded plus with qualifier", "pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com", "7.91+dfsg1"}, - {"tilde is not encoded", "pkg:deb/foo@1.0~rc1", "1.0~rc1"}, - {"no version", "pkg:npm/lodash", ""}, - {"invalid escape passed through", "pkg:npm/lodash@1.0%zz", "1.0%zz"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := VersionFromPURL(tt.purl); got != tt.want { - t.Errorf("VersionFromPURL(%q) = %q, want %q", tt.purl, got, tt.want) - } - v := &Version{PURL: tt.purl} - if got := v.Version(); got != tt.want { - t.Errorf("Version.Version() for %q = %q, want %q", tt.purl, got, tt.want) - } - }) - } -} - -// TestVersionEscapedVersion checks the value the templates put in a URL. It -// must survive the round trip back through the router: escaping here and -// decoding per path segment on the way in has to yield the original version. -func TestVersionEscapedVersion(t *testing.T) { - tests := []struct { - name string - purl string - want string - }{ - {"simple", "pkg:npm/lodash@4.17.21", "4.17.21"}, - // "+" is legal in a path segment, so it stays literal and the UI keeps - // showing the version the way Debian writes it. - {"plus stays literal", "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1-2ubuntu0.1"}, - // A slash would otherwise split the version into two path segments. - {"slash", "pkg:golang/example@release%2F1", "release%2F1"}, - // A question mark would otherwise start the query string. - {"question mark", "pkg:npm/example@v1%3Fbuild", "v1%3Fbuild"}, - // A version containing a literal "%2B" is stored double-encoded; the - // link must re-encode it or it decodes back to "+" instead. - {"literal percent escape", "pkg:npm/example@1.0%252B", "1.0%252B"}, - {"space", "pkg:npm/example@1.0%20beta", "1.0%20beta"}, - {"no version", "pkg:npm/lodash", ""}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - v := &Version{PURL: tt.purl} - got := v.EscapedVersion() - if got != tt.want { - t.Errorf("EscapedVersion() for %q = %q, want %q", tt.purl, got, tt.want) - } - // The router decodes each path segment, which must give back the - // version the page displays. - decoded, err := url.PathUnescape(got) - if err != nil { - t.Fatalf("PathUnescape(%q) failed: %v", got, err) - } - if decoded != v.Version() { - t.Errorf("round trip for %q = %q, want %q", tt.purl, decoded, v.Version()) - } - }) - } -} - -func TestVersionDisplayPURL(t *testing.T) { - tests := []struct { - name string - purl string - want string - }{ - {"simple", "pkg:npm/lodash@4.17.21", "pkg:npm/lodash@4.17.21"}, - { - "encoded plus", - "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", - "pkg:deb/nmap@7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", - }, - { - "qualifier preserved", - "pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com", - "pkg:deb/nmap@7.91+dfsg1?repository_url=http%3A%2F%2Fexample.com", - }, - // The namespace is encoded too: MakePURLString("npm", "@babel/core", …) - // produces "pkg:npm/%40babel/core@…". - {"encoded npm scope", "pkg:npm/%40babel/core@7.0.0", "pkg:npm/@babel/core@7.0.0"}, - {"encoded scope without version", "pkg:npm/%40babel/core", "pkg:npm/@babel/core"}, - {"no version", "pkg:npm/lodash", "pkg:npm/lodash"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - v := &Version{PURL: tt.purl} - if got := v.DisplayPURL(); got != tt.want { - t.Errorf("DisplayPURL() for %q = %q, want %q", tt.purl, got, tt.want) - } - }) - } -} - -// TestGetRecentlyCachedPackagesDecodesVersion guards the dashboard's "recently -// cached" list, which derives the version from the version PURL. -func TestGetRecentlyCachedPackagesDecodesVersion(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - const versionPURL = "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1" - - if err := db.UpsertPackage(&Package{ - PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap", - }); err != nil { - t.Fatalf("UpsertPackage failed: %v", err) - } - if err := db.UpsertVersion(&Version{ - PURL: versionPURL, PackagePURL: "pkg:deb/nmap", - }); err != nil { - t.Fatalf("UpsertVersion failed: %v", err) - } - if err := db.UpsertArtifact(&Artifact{ - VersionPURL: versionPURL, - Filename: "nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb", - UpstreamURL: "http://archive.ubuntu.com/ubuntu/pool/universe/n/nmap/nmap.deb", - StoragePath: sql.NullString{String: "/cache/nmap.deb", Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - }); err != nil { - t.Fatalf("UpsertArtifact failed: %v", err) - } - - recent, err := db.GetRecentlyCachedPackages(10) - if err != nil { - t.Fatalf("GetRecentlyCachedPackages failed: %v", err) - } - if len(recent) != 1 { - t.Fatalf("expected 1 recent package, got %d", len(recent)) - } - const want = "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1" - if recent[0].Version != want { - t.Errorf("Version = %q, want %q", recent[0].Version, want) - } - if recent[0].VersionPURL != versionPURL { - t.Errorf("VersionPURL = %q, want %q", recent[0].VersionPURL, versionPURL) - } - }) -} diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go deleted file mode 100644 index eea8ee3..0000000 --- a/internal/denylist/denylist.go +++ /dev/null @@ -1,157 +0,0 @@ -// Package denylist implements an immutable, exact-version package policy. -package denylist - -import ( - "fmt" - "strings" - - "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/purl" -) - -// Policy is safe for concurrent reads. A nil policy allows every version. -type Policy struct { - packages map[string]map[string]bool -} - -// New validates and canonicalizes versioned PURLs. Qualifiers and subpaths are -// rejected because the proxy cannot reliably distinguish them at every endpoint, -// except a conda channel, which download URLs always name. -func New(packages []string) (*Policy, error) { - if len(packages) == 0 { - return nil, nil - } - p := &Policy{packages: make(map[string]map[string]bool)} - for _, value := range packages { - pkg, err := purl.Parse(value) - if err != nil { - return nil, fmt.Errorf("invalid denylist package %q: %w", value, err) - } - if pkg.Version == "" || !allowedQualifiers(pkg) || pkg.Subpath != "" || - strings.ContainsAny(pkg.Namespace+pkg.Name+pkg.Version, "*?[]<>=| \t\r\n") { - return nil, fmt.Errorf("invalid denylist package %q: expected an exact versioned PURL without qualifiers or subpath", value) - } - id := identify(pkg) - key := id.key - if id.channel != "" { - key = id.channelKey() - } - if p.packages[key] == nil { - p.packages[key] = make(map[string]bool) - } - p.packages[key][id.version] = true - // Debian and RPM filenames leave the epoch out, so an entry with - // one also matches the version without it. - if pkg.Type == "deb" || pkg.Type == "rpm" { - if _, version, ok := strings.Cut(id.version, ":"); ok && version != "" { - p.packages[key][version] = true - } - } - } - return p, nil -} - -func allowedQualifiers(pkg *purl.PURL) bool { - for _, q := range pkg.Qualifiers { - if pkg.Type != "conda" || q.Key != "channel" { - return false - } - } - return true -} - -// Denied reports whether a versioned PURL is denied. -func (p *Policy) Denied(value string) bool { - if p == nil || len(p.packages) == 0 { - return false - } - pkg, err := purl.Parse(value) - if err != nil { - return false - } - id := identify(pkg) - versions := []string{id.version} - // A platform gem's version carries the platform ("1.16.0-x86_64-linux"), - // so also match the plain version. - if pkg.Type == "gem" { - if base, _, found := strings.Cut(id.version, "-"); found && base != "" { - versions = append(versions, base) - } - } - for _, version := range versions { - if p.packages[id.key][version] || (id.channel != "" && p.packages[id.channelKey()][version]) { - return true - } - } - return false -} - -// Versions returns a caller-owned set of denied versions for a package PURL. -func (p *Policy) Versions(packagePURL string) map[string]bool { - if p == nil || len(p.packages) == 0 { - return nil - } - pkg, err := purl.Parse(packagePURL) - if err != nil { - return nil - } - id := identify(pkg) - result := make(map[string]bool) - for version := range p.packages[id.key] { - result[version] = true - } - if id.channel != "" { - for version := range p.packages[id.channelKey()] { - result[version] = true - } - } - if len(result) == 0 { - return nil - } - return result -} - -// identity is the form a PURL is compared in: the same for denylist entries -// and for the PURLs handlers build, however each spells the package. -type identity struct { - key string // versionless PURL without channel - channel string // conda channel, if any - version string -} - -func (id identity) channelKey() string { - return id.key + "?channel=" + id.channel -} - -// genericSwiftRegistry returns the scope and name of a Swift registry archive, -// which the proxy stores as pkg:generic/swift-registry/{scope}.{name}. -func genericSwiftRegistry(pkg *purl.PURL) (scope, name string, ok bool) { - if pkg.Type != "generic" || !strings.EqualFold(pkg.Namespace, "swift-registry") { - return "", "", false - } - scope, name, found := strings.Cut(pkg.Name, ".") - return scope, name, found && scope != "" && name != "" -} - -// identify compares PURLs in purl's comparison form. The generic -// swift-registry spelling is the proxy's own, so it is mapped here first. -// The namespace of a distro type names the distribution, which handlers -// include for some types and not others, so it is dropped as retention and -// the inventory do. -func identify(pkg *purl.PURL) identity { - if scope, name, ok := genericSwiftRegistry(pkg); ok { - pkg = purl.New("swift", scope, name, pkg.Version, nil) - } - c := pkg.ComparisonForm() - var channel string - if c.Type == "conda" { - // An entry without a channel matches every channel. - channel = c.Qualifier("channel") - } - namespace := c.Namespace - if packageurl.DistroType(c.Type) { - namespace = "" - } - key := purl.New(c.Type, namespace, c.Name, "", nil).String() - return identity{key: packageurl.PolicyPackageKey(key), channel: channel, version: c.Version} -} diff --git a/internal/denylist/denylist_test.go b/internal/denylist/denylist_test.go deleted file mode 100644 index 03edc99..0000000 --- a/internal/denylist/denylist_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package denylist - -import "testing" - -func TestPolicy(t *testing.T) { - p, err := New([]string{"pkg:pypi/My_Package@1.0", "pkg:npm/@scope/name@2.0.0", "pkg:npm/%40scope/name@2.0.0"}) - if err != nil { - t.Fatal(err) - } - for _, value := range []string{"pkg:pypi/my-package@1.0", "pkg:npm/%40scope/name@2.0.0"} { - if !p.Denied(value) { - t.Errorf("not denied: %s", value) - } - } - for _, value := range []string{"pkg:pypi/my-package@1.1", "pkg:pypi/my-package", "pkg:npm/other@2.0.0", "invalid"} { - if p.Denied(value) { - t.Errorf("unexpected denial: %s", value) - } - } - versions := p.Versions("pkg:pypi/my-package") - delete(versions, "1.0") - if !p.Denied("pkg:pypi/my-package@1.0") { - t.Fatal("caller mutated policy") - } - var empty *Policy - if empty.Denied("pkg:npm/example@1") || empty.Versions("pkg:npm/example") != nil { - t.Fatal("nil policy must allow everything") - } -} - -func TestInvalidPolicy(t *testing.T) { - for _, value := range []string{"", "not-a-purl", "pkg:npm/foo", "pkg:npm/foo@*", "pkg:npm/foo@>=1", "pkg:npm/foo@1?arch=x86", "pkg:npm/foo@1#src", "pkg:conda/numpy@1?subdir=linux-64", "pkg:pypi/foo@1?channel=x"} { - t.Run(value, func(t *testing.T) { - if _, err := New([]string{value}); err == nil { - t.Fatal("invalid denylist accepted") - } - }) - } -} - -func TestEcosystemSpellings(t *testing.T) { - p, err := New([]string{ - "pkg:pypi/zope.interface@6.0", - "pkg:nuget/Newtonsoft.Json@13.0.1-Beta", - "pkg:gem/nokogiri@1.16.0", - "pkg:gem/sorbet-static@0.5.1-universal-darwin-22", - "pkg:conda/numpy@1.26.4", - "pkg:conda/scipy@1.0?channel=conda-forge", - "pkg:conda/bioconda/samtools@1.0", - "pkg:composer/symfony/console@6.1.7", - "pkg:composer/monolog/monolog@v3.5.0", - "pkg:oci/library/alpine@sha256:abc", - "pkg:oci/upstream%2Fghcr%2Forg%2Fimage@sha256:def", - "pkg:swift/Apple/Swift-NIO@2.0.0", - "pkg:generic/swift-registry/apple.example@1.0.0", - }) - if err != nil { - t.Fatal(err) - } - for _, value := range []string{ - "pkg:pypi/zope-interface@6.0", "pkg:pypi/Zope__Interface@6.0", - "pkg:nuget/newtonsoft.json@13.0.1-beta", - "pkg:gem/nokogiri@1.16.0", "pkg:gem/nokogiri@1.16.0-x86_64-linux", - "pkg:gem/sorbet-static@0.5.1-universal-darwin-22", - "pkg:conda/numpy@1.26.4", "pkg:conda/conda-forge%2Fnumpy@1.26.4", - "pkg:conda/conda-forge%2Fscipy@1.0", "pkg:conda/bioconda%2Fsamtools@1.0", - "pkg:composer/symfony/console@v6.1.7", "pkg:composer/symfony/console@V6.1.7", - "pkg:composer/monolog/monolog@3.5.0", - "pkg:oci/library%2Falpine@sha256:abc", "pkg:oci/library/alpine@sha256:abc", - "pkg:oci/upstream/ghcr/org/image@sha256:def", - "pkg:generic/swift-registry/apple.swift-nio@2.0.0", "pkg:swift/apple/swift-nio@2.0.0", - "pkg:swift/apple/example@1.0.0", - } { - if !p.Denied(value) { - t.Errorf("not denied: %s", value) - } - } - for _, value := range []string{ - "pkg:pypi/zopeinterface@6.0", "pkg:gem/nokogiri@1.16.1-x86_64-linux", "pkg:gem/Nokogiri@1.16.0", - "pkg:conda/bioconda%2Fscipy@1.0", "pkg:conda/scipy@1.0", "pkg:conda/conda-forge%2Fsamtools@1.0", - "pkg:composer/symfony/console@6.1.70", "pkg:composer/symfony/console@vv6.1.7", "pkg:npm/demo@v6.1.7", - "pkg:oci/alpine@sha256:abc", "pkg:oci/library%2Falpine@sha256:abd", - "pkg:generic/other/apple.swift-nio@2.0.0", "pkg:swift/github.com/apple/swift-nio@2.0.0", - } { - if p.Denied(value) { - t.Errorf("unexpected denial: %s", value) - } - } - if !p.Versions("pkg:pypi/zope_interface")["6.0"] { - t.Error("pypi versions not found under another spelling") - } -} - -// TestDistroNamespaces checks that the distribution namespace of a distro -// package is ignored on both sides, as handlers build some of these PURLs -// without one and some with one. -func TestDistroNamespaces(t *testing.T) { - p, err := New([]string{"pkg:deb/debian/curl@8.5.0-2", "pkg:rpm/fedora/curl@8.6.0-1.fc40", "pkg:apk/busybox@1.36.1-r15", "pkg:alpm/glibc@2.39-1"}) - if err != nil { - t.Fatal(err) - } - for _, value := range []string{"pkg:deb/curl@8.5.0-2", "pkg:rpm/curl@8.6.0-1.fc40", "pkg:apk/alpine/busybox@1.36.1-r15", "pkg:alpm/arch/glibc@2.39-1"} { - if !p.Denied(value) { - t.Errorf("not denied: %s", value) - } - } - if p.Denied("pkg:deb/curl@8.5.0-3") { - t.Error("other version denied") - } -} - -// TestDistroEpochs checks that a deb or rpm entry with an epoch matches the -// version as it appears in a filename, which leaves the epoch out. -func TestDistroEpochs(t *testing.T) { - p, err := New([]string{"pkg:deb/foo@1:2.3-1", "pkg:rpm/bar@2:1.0-1.fc40"}) - if err != nil { - t.Fatal(err) - } - for _, value := range []string{"pkg:deb/foo@2.3-1", "pkg:deb/foo@1:2.3-1", "pkg:rpm/bar@1.0-1.fc40"} { - if !p.Denied(value) { - t.Errorf("not denied: %s", value) - } - } - if p.Denied("pkg:deb/foo@2.3-2") { - t.Error("other version denied") - } -} diff --git a/internal/enrichment/enrichment.go b/internal/enrichment/enrichment.go index 2246e39..247dd2b 100644 --- a/internal/enrichment/enrichment.go +++ b/internal/enrichment/enrichment.go @@ -1,126 +1,303 @@ -// Package enrichment adapts github.com/git-pkgs/enrichment to the proxy's -// enrichment API. It builds package URLs from the proxy's ecosystem names, -// normalizes licenses for display and treats retracted versions as yanked. +// Package enrichment provides package metadata enrichment using external data sources. +// It fetches license information, vulnerability data, and version information +// from package registries and vulnerability databases. package enrichment import ( "context" - "errors" "log/slog" "sync" + "time" - shared "github.com/git-pkgs/enrichment" - "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" + _ "github.com/git-pkgs/registries/all" // Import all registry implementations "github.com/git-pkgs/spdx" + "github.com/git-pkgs/vers" + "github.com/git-pkgs/vulns" + "github.com/git-pkgs/vulns/osv" ) -// Service looks up package, version and vulnerability data. +// Service provides package enrichment capabilities. type Service struct { logger *slog.Logger - registries *shared.RegistriesClient - vulns *shared.VulnerabilityClient + regClient *registries.Client + vulnSource vulns.Source } // New creates a new enrichment service. func New(logger *slog.Logger) *Service { return &Service{ logger: logger, - registries: shared.NewRegistriesClient(), - vulns: shared.NewVulnerabilityClient(), + regClient: registries.DefaultClient(), + vulnSource: osv.New(), } } -type ( - PackageInfo = shared.PackageInfo - VersionInfo = shared.VersionInfo - VulnInfo = shared.VulnInfo -) +// PackageInfo contains enriched package metadata. +type PackageInfo struct { + Ecosystem string + Name string + LatestVersion string + License string + Description string + Homepage string + Repository string + RegistryURL string +} -// EnrichPackage fetches metadata for a package from its registry. It returns -// nil without an error when the registry doesn't have the package or the name -// can't be expressed as a PURL. +// VersionInfo contains enriched version metadata. +type VersionInfo struct { + Number string + License string + PublishedAt time.Time + Integrity string + Yanked bool +} + +// VulnInfo contains vulnerability information for a package version. +type VulnInfo struct { + ID string + Summary string + Severity string + CVSSScore float64 + FixedVersion string + References []string +} + +// EnrichPackage fetches metadata for a package from registry APIs. func (s *Service) EnrichPackage(ctx context.Context, ecosystem, name string) (*PackageInfo, error) { - purlStr := packageurl.MakeString(ecosystem, name, "") - if purlStr == "" { - return nil, nil - } - info, err := s.registries.GetPackage(ctx, purlStr) - if errors.Is(err, registries.ErrNotFound) { - return nil, nil - } - if err != nil || info == nil { + purlStr := purl.MakePURLString(ecosystem, name, "") + + pkg, err := registries.FetchPackageFromPURL(ctx, purlStr, s.regClient) + if err != nil { return nil, err } - info.Ecosystem = ecosystem - info.License = normalizeLicense(info.License) + + if pkg == nil { + return nil, nil + } + + info := &PackageInfo{ + Ecosystem: ecosystem, + Name: pkg.Name, + LatestVersion: pkg.LatestVersion, + Description: pkg.Description, + Homepage: pkg.Homepage, + Repository: pkg.Repository, + RegistryURL: registries.DefaultURL(ecosystem), + } + + // Normalize license to SPDX format + if pkg.Licenses != "" { + if normalized, err := spdx.NormalizeExpressionLax(pkg.Licenses); err == nil { + info.License = normalized + } else { + info.License = pkg.Licenses + } + } + return info, nil } // EnrichVersion fetches metadata for a specific package version. func (s *Service) EnrichVersion(ctx context.Context, ecosystem, name, version string) (*VersionInfo, error) { - purlStr := packageurl.MakeString(ecosystem, name, version) - if purlStr == "" { - return nil, nil - } - info, err := s.registries.GetVersion(ctx, purlStr) - if err != nil || info == nil { + purlStr := purl.MakePURLString(ecosystem, name, version) + + ver, err := registries.FetchVersionFromPURL(ctx, purlStr, s.regClient) + if err != nil { return nil, err } - adaptVersion(info) + + if ver == nil { + return nil, nil + } + + info := &VersionInfo{ + Number: ver.Number, + PublishedAt: ver.PublishedAt, + Integrity: ver.Integrity, + Yanked: ver.Status == registries.StatusYanked || ver.Status == registries.StatusRetracted, + } + + // Normalize license + if ver.Licenses != "" { + if normalized, err := spdx.NormalizeExpressionLax(ver.Licenses); err == nil { + info.License = normalized + } else { + info.License = ver.Licenses + } + } + return info, nil } -// adaptVersion reports retracted versions as yanked and normalizes the license. -func adaptVersion(info *VersionInfo) { - if info.Status == string(registries.StatusRetracted) { - info.Yanked = true - } - info.License = normalizeLicense(info.License) -} - -// BulkEnrichPackages fetches metadata for multiple packages in parallel, -// keyed by PURL. Packages that can't be fetched are left out. +// BulkEnrichPackages fetches metadata for multiple packages in parallel. func (s *Service) BulkEnrichPackages(ctx context.Context, packages []struct{ Ecosystem, Name string }) map[string]*PackageInfo { - purls := make([]string, 0, len(packages)) - for _, pkg := range packages { - if purlStr := packageurl.MakeString(pkg.Ecosystem, pkg.Name, ""); purlStr != "" { - purls = append(purls, purlStr) + purls := make([]string, len(packages)) + for i, pkg := range packages { + purls[i] = purl.MakePURLString(pkg.Ecosystem, pkg.Name, "") + } + + pkgData := registries.BulkFetchPackages(ctx, purls, s.regClient) + + result := make(map[string]*PackageInfo, len(pkgData)) + for purlStr, pkg := range pkgData { + if pkg == nil { + continue } + + p, _ := purl.Parse(purlStr) + info := &PackageInfo{ + Ecosystem: p.Type, + Name: pkg.Name, + LatestVersion: pkg.LatestVersion, + Description: pkg.Description, + Homepage: pkg.Homepage, + Repository: pkg.Repository, + RegistryURL: registries.DefaultURL(p.Type), + } + + if pkg.Licenses != "" { + if normalized, err := spdx.NormalizeExpressionLax(pkg.Licenses); err == nil { + info.License = normalized + } else { + info.License = pkg.Licenses + } + } + + result[purlStr] = info } - if len(purls) == 0 { - return map[string]*PackageInfo{} - } - result, _ := s.registries.BulkLookup(ctx, purls) - for _, info := range result { - info.License = normalizeLicense(info.License) - } + return result } // CheckVulnerabilities queries for vulnerabilities affecting a package version. func (s *Service) CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]VulnInfo, error) { - return s.vulns.Check(ctx, ecosystem, name, version) + p := purl.MakePURL(ecosystem, name, version) + + vulnList, err := s.vulnSource.Query(ctx, p) + if err != nil { + return nil, err + } + + results := make([]VulnInfo, 0, len(vulnList)) + for _, v := range vulnList { + info := VulnInfo{ + ID: v.ID, + Summary: v.Summary, + Severity: v.SeverityLevel(), + CVSSScore: v.CVSSScore(), + FixedVersion: v.FixedVersion(ecosystem, name), + } + + for _, ref := range v.References { + info.References = append(info.References, ref.URL) + } + + results = append(results, info) + } + + return results, nil +} + +// BulkCheckVulnerabilities queries vulnerabilities for multiple package versions. +func (s *Service) BulkCheckVulnerabilities(ctx context.Context, packages []struct{ Ecosystem, Name, Version string }) (map[string][]VulnInfo, error) { + purls := make([]*purl.PURL, len(packages)) + for i, pkg := range packages { + purls[i] = purl.MakePURL(pkg.Ecosystem, pkg.Name, pkg.Version) + } + + vulnResults, err := s.vulnSource.QueryBatch(ctx, purls) + if err != nil { + return nil, err + } + + result := make(map[string][]VulnInfo, len(packages)) + for i, vulnList := range vulnResults { + pkg := packages[i] + key := purl.MakePURLString(pkg.Ecosystem, pkg.Name, pkg.Version) + + var infos []VulnInfo + for _, v := range vulnList { + info := VulnInfo{ + ID: v.ID, + Summary: v.Summary, + Severity: v.SeverityLevel(), + CVSSScore: v.CVSSScore(), + FixedVersion: v.FixedVersion(pkg.Ecosystem, pkg.Name), + } + for _, ref := range v.References { + info.References = append(info.References, ref.URL) + } + infos = append(infos, info) + } + result[key] = infos + } + + return result, nil +} + +// IsOutdated checks if a version is older than the latest version. +func (s *Service) IsOutdated(currentVersion, latestVersion string) bool { + if latestVersion == "" || currentVersion == "" { + return false + } + return vers.Compare(currentVersion, latestVersion) < 0 } // GetLatestVersion fetches the latest version for a package. func (s *Service) GetLatestVersion(ctx context.Context, ecosystem, name string) (string, error) { - info, err := s.EnrichPackage(ctx, ecosystem, name) - if err != nil || info == nil { + purlStr := purl.MakePURLString(ecosystem, name, "") + + latest, err := registries.FetchLatestVersionFromPURL(ctx, purlStr, s.regClient) + if err != nil { return "", err } - return info.LatestVersion, nil + + if latest == nil { + return "", nil + } + + return latest.Number, nil } -// normalizeLicense returns license as an SPDX expression when it can be -// parsed as one, and unchanged otherwise. -func normalizeLicense(license string) string { +// LicenseCategory represents the category of a license. +type LicenseCategory string + +const ( + LicensePermissive LicenseCategory = "permissive" + LicenseCopyleft LicenseCategory = "copyleft" + LicenseUnknown LicenseCategory = "unknown" +) + +// CategorizeLicense returns the category of a license. +func (s *Service) CategorizeLicense(license string) LicenseCategory { + if license == "" || license == "Unknown" { + return LicenseUnknown + } + + if spdx.HasCopyleft(license) { + return LicenseCopyleft + } + + if spdx.IsFullyPermissive(license) { + return LicensePermissive + } + + return LicenseUnknown +} + +// NormalizeLicense normalizes a license string to SPDX format. +func (s *Service) NormalizeLicense(license string) string { if license == "" { return "" } + if normalized, err := spdx.NormalizeExpressionLax(license); err == nil { return normalized } + return license } @@ -130,25 +307,40 @@ type EnrichmentResult struct { Version *VersionInfo Vulnerabilities []VulnInfo IsOutdated bool - LicenseCategory shared.LicenseCategory + LicenseCategory LicenseCategory } -// EnrichFull fetches package, version and vulnerability data for a package -// version concurrently. Unsupported vulnerability identities return an error; -// other failed lookups are logged and left empty. +// EnrichFull performs full enrichment for a package version. func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version string) (*EnrichmentResult, error) { result := &EnrichmentResult{} var wg sync.WaitGroup var pkgErr, verErr, vulnErr error - wg.Go(func() { result.Package, pkgErr = s.EnrichPackage(ctx, ecosystem, name) }) - wg.Go(func() { result.Version, verErr = s.EnrichVersion(ctx, ecosystem, name, version) }) - wg.Go(func() { result.Vulnerabilities, vulnErr = s.CheckVulnerabilities(ctx, ecosystem, name, version) }) - wg.Wait() - if errors.Is(vulnErr, shared.ErrUnsupportedVulnerabilityIdentity) { - return nil, vulnErr - } + // Fetch package info + wg.Add(1) + go func() { + defer wg.Done() + result.Package, pkgErr = s.EnrichPackage(ctx, ecosystem, name) + }() + + // Fetch version info + wg.Add(1) + go func() { + defer wg.Done() + result.Version, verErr = s.EnrichVersion(ctx, ecosystem, name, version) + }() + + // Check vulnerabilities + wg.Add(1) + go func() { + defer wg.Done() + result.Vulnerabilities, vulnErr = s.CheckVulnerabilities(ctx, ecosystem, name, version) + }() + + wg.Wait() + + // Log errors but don't fail if pkgErr != nil { s.logger.Debug("failed to enrich package", "ecosystem", ecosystem, "name", name, "error", pkgErr) } @@ -159,17 +351,19 @@ func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version strin s.logger.Debug("failed to check vulnerabilities", "ecosystem", ecosystem, "name", name, "version", version, "error", vulnErr) } - if result.Package != nil { - result.IsOutdated = shared.IsOutdated(version, result.Package.LatestVersion) + // Determine outdated status + if result.Package != nil && result.Package.LatestVersion != "" { + result.IsOutdated = s.IsOutdated(version, result.Package.LatestVersion) } + // Categorize license license := "" if result.Version != nil && result.Version.License != "" { license = result.Version.License - } else if result.Package != nil { + } else if result.Package != nil && result.Package.License != "" { license = result.Package.License } - result.LicenseCategory = shared.CategorizeLicense(license) + result.LicenseCategory = s.CategorizeLicense(license) return result, nil } diff --git a/internal/enrichment/enrichment_test.go b/internal/enrichment/enrichment_test.go index f01bb29..aa9a16e 100644 --- a/internal/enrichment/enrichment_test.go +++ b/internal/enrichment/enrichment_test.go @@ -1,14 +1,9 @@ package enrichment import ( - "context" - "errors" "log/slog" "os" "testing" - - shared "github.com/git-pkgs/enrichment" - "github.com/git-pkgs/registries" ) func TestNew(t *testing.T) { @@ -19,65 +14,85 @@ func TestNew(t *testing.T) { t.Fatal("New() returned nil") } - if svc.registries == nil { - t.Error("registries is nil") + if svc.regClient == nil { + t.Error("regClient is nil") } - if svc.vulns == nil { - t.Error("vulns is nil") + if svc.vulnSource == nil { + t.Error("vulnSource is nil") } } -func TestSwiftRegistryIdentityLookups(t *testing.T) { - svc := New(slog.New(slog.NewTextHandler(os.Stdout, nil))) - ctx := context.Background() +func TestIsOutdated(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + svc := New(logger) - packageInfo, err := svc.EnrichPackage(ctx, "swift", "apple/example") - if err != nil || packageInfo != nil { - t.Errorf("EnrichPackage() = %#v, %v; want nil, nil", packageInfo, err) - } - - versionInfo, err := svc.EnrichVersion(ctx, "swift", "apple/example", "1.2.3") - if err != nil || versionInfo != nil { - t.Errorf("EnrichVersion() = %#v, %v; want nil, nil", versionInfo, err) - } - - vulnerabilities, err := svc.CheckVulnerabilities(ctx, "swift", "apple/example", "1.2.3") - if !errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) || vulnerabilities != nil { - t.Errorf("CheckVulnerabilities() = %#v, %v; want unsupported identity", vulnerabilities, err) - } - if _, err := svc.EnrichFull(ctx, "swift", "apple/example", "1.2.3"); !errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { - t.Errorf("EnrichFull error = %v, want unsupported identity", err) - } - - latest, err := svc.GetLatestVersion(ctx, "swift", "apple/example") - if err != nil || latest != "" { - t.Errorf("GetLatestVersion() = %q, %v; want empty string, nil", latest, err) - } - - packages := []struct{ Ecosystem, Name string }{{Ecosystem: "swift", Name: "apple/example"}} - if got := svc.BulkEnrichPackages(ctx, packages); len(got) != 0 { - t.Errorf("BulkEnrichPackages() = %#v, want empty result", got) - } -} - -func TestAdaptVersion(t *testing.T) { tests := []struct { - status registries.VersionStatus - yanked bool - license string - wantYanked bool - wantLicense string + current string + latest string + expected bool }{ - {status: registries.StatusRetracted, license: "mit", wantYanked: true, wantLicense: "MIT"}, - {status: registries.StatusYanked, yanked: true, wantYanked: true}, - {status: registries.StatusDeprecated, license: "Custom terms", wantLicense: "Custom terms"}, + {"1.0.0", "2.0.0", true}, + {"2.0.0", "2.0.0", false}, + {"2.0.0", "1.0.0", false}, + {"1.0.0", "", false}, + {"", "2.0.0", false}, + {"1.2.3", "1.2.4", true}, + {"1.2.4", "1.2.3", false}, } + for _, tc := range tests { - info := &VersionInfo{Status: string(tc.status), Yanked: tc.yanked, License: tc.license} - adaptVersion(info) - if info.Yanked != tc.wantYanked || info.License != tc.wantLicense { - t.Errorf("adaptVersion(%s, %q) = yanked %v, license %q; want %v, %q", tc.status, tc.license, info.Yanked, info.License, tc.wantYanked, tc.wantLicense) + result := svc.IsOutdated(tc.current, tc.latest) + if result != tc.expected { + t.Errorf("IsOutdated(%q, %q) = %v, want %v", tc.current, tc.latest, result, tc.expected) + } + } +} + +func TestCategorizeLicense(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + svc := New(logger) + + tests := []struct { + license string + expected LicenseCategory + }{ + {"MIT", LicensePermissive}, + {"Apache-2.0", LicensePermissive}, + {"BSD-3-Clause", LicensePermissive}, + {"GPL-3.0", LicenseCopyleft}, + {"AGPL-3.0", LicenseCopyleft}, + {"LGPL-2.1", LicenseCopyleft}, + {"", LicenseUnknown}, + {"Unknown", LicenseUnknown}, + } + + for _, tc := range tests { + result := svc.CategorizeLicense(tc.license) + if result != tc.expected { + t.Errorf("CategorizeLicense(%q) = %v, want %v", tc.license, result, tc.expected) + } + } +} + +func TestNormalizeLicense(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + svc := New(logger) + + tests := []struct { + input string + expected string + }{ + {"MIT", "MIT"}, + {"Apache 2", "Apache-2.0"}, + {"Apache-2.0", "Apache-2.0"}, + {"", ""}, + } + + for _, tc := range tests { + result := svc.NormalizeLicense(tc.input) + if result != tc.expected { + t.Errorf("NormalizeLicense(%q) = %q, want %q", tc.input, result, tc.expected) } } } diff --git a/internal/gitcache/budget.go b/internal/gitcache/budget.go deleted file mode 100644 index 2b0c228..0000000 --- a/internal/gitcache/budget.go +++ /dev/null @@ -1,139 +0,0 @@ -package gitcache - -import ( - "context" - "errors" - "sync/atomic" - "time" -) - -const ( - // maxReserveChunk caps the space a fetch reserves ahead of what it has - // received, so the cache is not locked for every read. Each fetch takes - // at most a sixteenth of MaxSize ahead, so fetches share a small cache. - maxReserveChunk = 4 << 20 - reserveChunks = 16 - // cutWait bounds how long a fetch waits for one that was cut off to - // give back its space. - cutWait = time.Minute -) - -// reservation holds space within MaxSize for the bytes a fetch receives, -// until they are counted in the repository's size. Fetches still running -// therefore count toward MaxSize, so the cache stays within it however many -// run at once. -type reservation struct { - c *Cache - // reserved is written under c.mu; reads that find it large enough skip - // the lock. - reserved atomic.Int64 - // cut, guarded by c.mu, marks a fetch stopped because the cache could - // not make room for it. - cut bool - // ended, guarded by c.mu, marks a reservation given back. A read - // relayed after that, by a git helper the fetch left behind, reserves - // nothing. - ended bool -} - -// errReservationEnded is returned by cover once the fetch has ended. -var errReservationEnded = errors.New("fetch has ended") - -func (c *Cache) newReservation() *reservation { - return &reservation{c: c} -} - -// cover reserves space for a fetch that has received relayed bytes, -// evicting idle repositories as needed. It returns ErrCacheFull when evicting -// every idle repository would not make room, unless another fetch has just -// been cut off, whose space it waits for: one cut off waits for nothing, so -// two fetches that do not fit together do not both fail. -func (r *reservation) cover(ctx context.Context, relayed int64) error { - c := r.c - if c.opts.MaxSize <= 0 || relayed <= r.reserved.Load() { - return nil - } - var timeout <-chan time.Time - for { - c.mu.Lock() - if r.ended { - c.mu.Unlock() - return errReservationEnded - } - if r.cut { - c.mu.Unlock() - return ErrCacheFull - } - need := relayed - r.reserved.Load() - removed, total := c.evictLocked(need) - fits := total+need <= c.opts.MaxSize - var freed chan struct{} - switch { - case fits: - chunk := min(maxReserveChunk, c.opts.MaxSize/reserveChunks) - n := min(need+chunk, c.opts.MaxSize-total) - r.reserved.Add(n) - c.reserved += n - case c.cutPending > 0: - freed = c.freed - default: - r.cut = true - c.cutPending++ - } - c.mu.Unlock() - c.removeEvicted(removed) - if fits { - return nil - } - if freed == nil { - return ErrCacheFull - } - if timeout == nil { - timeout = time.After(cutWait) - } - select { - case <-freed: - case <-timeout: - c.mu.Lock() - r.cut = true - c.cutPending++ - c.mu.Unlock() - return ErrCacheFull - case <-ctx.Done(): - return ctx.Err() - } - } -} - -// settle counts what the fetch left in the repository's size and gives back -// its reservation together, so the bytes are never counted twice or not at -// all. -func (r *reservation) settle(rp *repo, size int64) { - r.c.mu.Lock() - rp.size.Store(size) - r.releaseLocked() - r.c.mu.Unlock() -} - -// release gives back the reservation of a fetch that left nothing behind. -// It may follow settle. -func (r *reservation) release() { - r.c.mu.Lock() - r.releaseLocked() - r.c.mu.Unlock() -} - -func (r *reservation) releaseLocked() { - c := r.c - r.ended = true - n := r.reserved.Swap(0) - if r.cut { - r.cut = false - c.cutPending-- - } else if n == 0 { - return - } - c.reserved -= n - close(c.freed) - c.freed = make(chan struct{}) -} diff --git a/internal/gitcache/budget_test.go b/internal/gitcache/budget_test.go deleted file mode 100644 index 8ef58fa..0000000 --- a/internal/gitcache/budget_test.go +++ /dev/null @@ -1,311 +0,0 @@ -package gitcache - -import ( - "bytes" - "context" - "crypto/rand" - "errors" - "io" - "net/http" - "net/http/httptest" - "net/http/httputil" - "net/url" - "os" - "path/filepath" - "sync" - "testing" - "time" -) - -// cachedRepo adds a repository of the given size to the cache, still in use -// when busy. -func cachedRepo(t *testing.T, tc *testCache, name string, size int64, busy bool) *repo { - t.Helper() - rp := tc.acquire("up/" + name + ".git") - if err := os.MkdirAll(rp.dir, dirPermissions); err != nil { - t.Fatal(err) - } - rp.size.Store(size) - if !busy { - tc.release(rp) - } - return rp -} - -func (tc *testCache) budget() (reserved int64, cutPending int) { - tc.mu.Lock() - defer tc.mu.Unlock() - return tc.reserved, tc.cutPending -} - -func TestCoverEvictsIdleRepositories(t *testing.T) { - tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) - old := cachedRepo(t, tc, "old", 60, false) - busy := cachedRepo(t, tc, "busy", 30, true) - - res := tc.newReservation() - if err := res.cover(t.Context(), 20); err != nil { - t.Fatal(err) - } - if fileExists(old.dir) { - t.Error("idle repository was not evicted") - } - if !fileExists(busy.dir) { - t.Error("evicted a repository in use") - } - if reserved, _ := tc.budget(); reserved < 20 { - t.Errorf("reserved %d, want at least 20", reserved) - } - res.release() - if reserved, _ := tc.budget(); reserved != 0 { - t.Errorf("reserved %d after release", reserved) - } -} - -func TestCoverCutsOffOnceOnlyRepositoriesInUseAreLeft(t *testing.T) { - tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) - busy := cachedRepo(t, tc, "busy", 90, true) - - res := tc.newReservation() - if err := res.cover(t.Context(), 20); !errors.Is(err, ErrCacheFull) { - t.Fatalf("cover: %v, want ErrCacheFull", err) - } - if !fileExists(busy.dir) { - t.Error("evicted a repository in use") - } - res.release() - if reserved, cutPending := tc.budget(); reserved != 0 || cutPending != 0 { - t.Errorf("reserved %d, cut off %d after release", reserved, cutPending) - } -} - -// TestCoverKeepsIdleRepositoriesItCannotUse cuts off a fetch that would not -// fit even with every idle repository evicted, so none is. -func TestCoverKeepsIdleRepositoriesItCannotUse(t *testing.T) { - tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) - idle := cachedRepo(t, tc, "idle", 5, false) - cachedRepo(t, tc, "busy", 90, true) - - res := tc.newReservation() - if err := res.cover(t.Context(), 20); !errors.Is(err, ErrCacheFull) { - t.Fatalf("cover: %v, want ErrCacheFull", err) - } - if !fileExists(idle.dir) { - t.Error("evicted an idle repository without making room") - } - res.release() -} - -// TestCoverWaitsForCutOffFetch runs two fetches that do not fit together. -// Once one is cut off, the other waits for its space instead of failing too. -func TestCoverWaitsForCutOffFetch(t *testing.T) { - tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) - cachedRepo(t, tc, "busy", 40, true) - - first, second := tc.newReservation(), tc.newReservation() - if err := first.cover(t.Context(), 30); err != nil { - t.Fatal(err) - } - if err := second.cover(t.Context(), 20); err != nil { - t.Fatal(err) - } - if err := first.cover(t.Context(), 40); !errors.Is(err, ErrCacheFull) { - t.Fatalf("first: %v, want ErrCacheFull", err) - } - - done := make(chan error, 1) - go func() { done <- second.cover(context.Background(), 30) }() - select { - case err := <-done: - t.Fatalf("second did not wait for the first to be released: %v", err) - case <-time.After(100 * time.Millisecond): - } - first.release() - if err := <-done; err != nil { - t.Fatalf("second: %v", err) - } - second.release() - if reserved, cutPending := tc.budget(); reserved != 0 || cutPending != 0 { - t.Errorf("reserved %d, cut off %d after release", reserved, cutPending) - } -} - -// TestConcurrentClonesStayWithinMaxSize clones two repositories at once that -// do not fit in MaxSize together. Both are kept in use, so neither can be -// evicted: one clone succeeds, and the other is cut off without being -// marked too large. -func TestConcurrentClonesStayWithinMaxSize(t *testing.T) { - upstream := newFakeUpstream(t) - for _, name := range []string{"a", "b"} { - work := upstream.newRepo(name) - big := make([]byte, 256<<10) - _, _ = rand.Read(big) - if err := os.WriteFile(filepath.Join(work, "big.bin"), big, 0o600); err != nil { - t.Fatal(err) - } - runGit(t, work, "add", "big.bin") - runGit(t, work, "commit", "--quiet", "-m", "big") - runGit(t, work, "push", "--quiet", "origin", "main") - } - size := repoSize(t, upstream, "a") - maxSize := size * 5 / 3 - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = maxSize }) - for _, name := range []string{"a", "b"} { - defer tc.release(tc.acquire("up/" + name + ".git")) - } - - // Sample what the cache counts while the clones run. - var peak int64 - stop := make(chan struct{}) - sampled := make(chan struct{}) - go func() { - defer close(sampled) - for { - tc.mu.Lock() - total := tc.reserved - for _, rp := range tc.repos { - total += rp.size.Load() - } - tc.mu.Unlock() - peak = max(peak, total) - select { - case <-stop: - return - case <-time.After(100 * time.Microsecond): - } - } - }() - - // Held back, both fetches are under way before either receives a pack. - upstream.delayFetches.Store(int64(200 * time.Millisecond)) - statuses := make(map[string]int) - var mu sync.Mutex - var wg sync.WaitGroup - for _, name := range []string{"a", "b"} { - wg.Go(func() { - resp, err := http.Get(tc.url(name) + "/info/refs?service=git-upload-pack") - if err != nil { - t.Error(err) - return - } - _ = resp.Body.Close() - mu.Lock() - statuses[name] = resp.StatusCode - mu.Unlock() - }) - } - wg.Wait() - close(stop) - <-sampled - - if (statuses["a"] == http.StatusOK) == (statuses["b"] == http.StatusOK) { - t.Fatalf("statuses %v, want one 200 and one 507", statuses) - } - for name, status := range statuses { - if status != http.StatusOK && status != http.StatusInsufficientStorage { - t.Errorf("%s: status %d, want 507", name, status) - } - if tc.isTooLarge("up/" + name + ".git") { - t.Errorf("%s was marked too large", name) - } - } - // What a clone writes beyond the bytes it receives, such as its index, - // is counted only once it is measured. - if slack := size / 4; peak > maxSize+slack { - t.Errorf("cache counted %d, past max_size %d", peak, maxSize) - } - if reserved, cutPending := tc.budget(); reserved != 0 || cutPending != 0 { - t.Errorf("reserved %d, cut off %d once the clones ended", reserved, cutPending) - } - if entries, _ := os.ReadDir(tc.tmpDir()); len(entries) != 0 { - t.Errorf("left %d entries in the temporary directory", len(entries)) - } -} - -// TestCoverAfterReleaseReservesNothing has a relayed read arrive after the -// fetch gave back its reservation, as one from an orphaned git helper would. -func TestCoverAfterReleaseReservesNothing(t *testing.T) { - tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) - res := tc.newReservation() - if err := res.cover(t.Context(), 10); err != nil { - t.Fatal(err) - } - res.release() - ctx, cancel := context.WithCancel(t.Context()) - cancel() - _ = res.cover(ctx, 20) - if reserved, _ := tc.budget(); reserved != 0 { - t.Errorf("reserved %d after release", reserved) - } -} - -// TestCancelledFetchStopsRelaying cancels a fetch while the upstream is -// midway through sending the pack: the upstream connection is closed, and -// nothing stays reserved when the rest arrives. -func TestCancelledFetchStopsRelaying(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - upstreamURL, err := url.Parse(upstream.server.URL) - if err != nil { - t.Fatal(err) - } - arrived := make(chan struct{}) - release := make(chan struct{}) - result := make(chan string, 1) - forward := httputil.NewSingleHostReverseProxy(upstreamURL) - front := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - r.Body = io.NopCloser(bytes.NewReader(body)) - if r.Method != http.MethodPost || !bytes.Contains(body, []byte("command=fetch")) { - forward.ServeHTTP(w, r) - return - } - rec := httptest.NewRecorder() - forward.ServeHTTP(rec, r.Clone(context.Background())) - pack := rec.Body.Bytes() - w.Header().Set("Content-Type", rec.Header().Get("Content-Type")) - _, _ = w.Write(pack[:len(pack)/2]) - w.(http.Flusher).Flush() - close(arrived) - select { - case <-r.Context().Done(): - result <- "closed" - case <-release: - _, _ = w.Write(pack[len(pack)/2:]) - w.(http.Flusher).Flush() - result <- "sent" - } - })) - defer front.Close() - defer close(release) - tc := newTestCache(t, front.URL, func(o *Options) { o.MaxSize = 1 << 30 }) - - dir := t.TempDir() - if _, err := tc.run(t.Context(), dir, "init", "--quiet", "--bare"); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithCancel(t.Context()) - res := tc.newReservation() - done := make(chan error, 1) - go func() { done <- tc.fetch(ctx, dir, front.URL+"/repo", 0, res, nil, branchesAndTags...) }() - <-arrived - waitFor(t, func() bool { reserved, _ := tc.budget(); return reserved > 0 }) - cancel() - if err := <-done; err == nil { - t.Error("cancelled fetch succeeded") - } - res.release() - - select { - case <-result: - case <-time.After(10 * time.Second): - t.Error("upstream connection still open after the fetch was cancelled") - release <- struct{}{} - <-result - } - // Give a relay still running the time to read what the upstream sent. - time.Sleep(200 * time.Millisecond) - if reserved, _ := tc.budget(); reserved != 0 { - t.Errorf("reserved %d after the fetch ended", reserved) - } -} diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go deleted file mode 100644 index 54c97d0..0000000 --- a/internal/gitcache/cache.go +++ /dev/null @@ -1,544 +0,0 @@ -// Package gitcache serves Git repositories read-only over smart HTTP from a -// local cache of bare repositories, refreshed from configured upstreams. -package gitcache - -import ( - "cmp" - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "io/fs" - "log/slog" - "net/http" - "os" - "os/exec" - "path/filepath" - "regexp" - "runtime" - "slices" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/git-pkgs/proxy/internal/metrics" -) - -const ( - // tmpDirName holds clones in progress and evicted repositories being - // deleted. Upstream names cannot start with a dot, so it cannot clash - // with an upstream's directory, and the name is specific in case - // cache_dir is shared with other data. - tmpDirName = ".gitcache-tmp" - // lockFileName is locked while the cache is open, so a second process - // given the same cache_dir fails to start instead of deleting the first - // one's clones in progress. - lockFileName = ".gitcache-lock" - // usedStampInterval limits how often a repository's last use is written - // to disk. - usedStampInterval = time.Minute - // defaultLowSpeedTime is how many seconds a git transfer may receive - // nothing before it is abandoned. Fetches run with --quiet, so the - // upstream sends nothing while it counts objects; this is far longer. - defaultLowSpeedTime = 600 - ecosystem = "git" - upstreamHashLen = 8 - checkedStamp = "proxy-checked" - usedStamp = "proxy-used" - dirPermissions = 0o750 - lockFilePermissions = 0o600 -) - -// Options configures a Cache. -type Options struct { - CacheDir string - MaxSize int64 - MaxRepoSize int64 - RefreshWait time.Duration - StaleMaxAge time.Duration - // Client makes every upstream request, including those of the git - // processes the cache runs. Redirects are never followed. - Client *http.Client - Logger *slog.Logger - // Upstreams maps each configured upstream's name to its directory in - // CacheDir, as UpstreamDir names it. Only those directories are loaded - // at startup. A directory UpstreamDir would name for one of these names - // at another URL, left by repointing the upstream, is removed. Nothing - // else in CacheDir is touched, so it may hold other data. - Upstreams map[string]string - - // refCheckTimeout, lowSpeedTime, maxRequestBody, maxUploadPacks and - // maxRefListings override their defaults in tests, and refreshWaitOver, - // when set, replaces the RefreshWait timer. - refCheckTimeout time.Duration - refreshWaitOver <-chan time.Time - lowSpeedTime int - maxRequestBody int64 - maxUploadPacks int - maxRefListings int -} - -// Cache is the Git repository cache. internal/handler serves it at /git. -type Cache struct { - opts Options - client *http.Client - logger *slog.Logger - gitPath string - dir string - lock *os.File - relay *relay - fetches chan struct{} - uploadPacks chan struct{} - refListings chan struct{} - refCheckTimeout time.Duration - lowSpeedTime int - maxRequestBody int64 - // foldCase is set where the cache directory ignores case, so keys that - // differ only in case, which name one directory, are one repository. - foldCase bool - - ctx context.Context - cancel context.CancelFunc - // refreshes counts the refreshes running, which Close waits for. - refreshes sync.WaitGroup - - mu sync.Mutex - // closed stops refreshes from starting once Close has begun. - closed bool - repos map[string]*repo - tooLarge map[string]time.Time - // overBudget records that eviction left the cache over MaxSize because - // the rest of it was in use. - overBudget bool - // reserved is the space reservations hold for fetches in progress, and - // cutPending counts those cut off and not yet released. freed is closed - // and replaced whenever a reservation gives back space or one cut off is - // released. - reserved int64 - cutPending int - freed chan struct{} - // failedWants records, by repository key and object ID, wants the - // upstream could not provide. - failedWants map[string]time.Time -} - -// repo is a cached repository, or one being cloned. users, refreshing, -// refused and the map entry itself are guarded by Cache.mu. -type repo struct { - key string - dir string - - users int - refreshing *refreshCall - // refused is the error of the last refresh when the upstream reported - // the repository missing or refused it, until a ref check succeeds. - refused error - // writes serializes git fetches into the repository. - writes chan struct{} - - size atomic.Int64 - lastUsed atomic.Int64 - lastChecked atomic.Int64 - usedWritten atomic.Int64 -} - -var errLocked = errors.New("locked by another process") - -// maxConcurrentFetches caps clones and fetches running at once. Refreshes -// outlive the requests that start them, so without a cap a client could -// start any number and hang up. -const maxConcurrentFetches = 16 - -// uploadPacksPerCPU caps the upload-pack processes building packs at once, -// so a burst of clones cannot run the proxy out of memory. Each uses one -// delta-search thread (servingConfig); twice the CPUs keeps them busy while -// some packs wait on slow clients. -const uploadPacksPerCPU = 2 - -// refListingsPerCPU caps the upload-pack processes listing refs at once, for -// info/refs and ls-refs. They build no pack and so do not wait behind -// clones, but each is still a process. -const refListingsPerCPU = 4 - -// New creates the cache, loading repositories already in CacheDir. -func New(opts Options) (*Cache, error) { - gitPath, err := exec.LookPath("git") - if err != nil { - return nil, fmt.Errorf("upstream.git needs git installed: %w", err) - } - dir, err := filepath.Abs(opts.CacheDir) - if err != nil { - return nil, err - } - if err := os.MkdirAll(dir, dirPermissions); err != nil { - return nil, err - } - lock, err := lockFile(filepath.Join(dir, lockFileName)) - if errors.Is(err, errLocked) { - return nil, fmt.Errorf("git.cache_dir %s is in use by another process; each needs its own", dir) - } - if err != nil { - return nil, fmt.Errorf("locking git.cache_dir %s: %w", dir, err) - } - logger := opts.Logger - if logger == nil { - logger = slog.New(slog.DiscardHandler) - } - // A repository larger than the whole cache would be evicted once - // released and cloned again by the next request. - if opts.MaxSize > 0 && (opts.MaxRepoSize <= 0 || opts.MaxRepoSize > opts.MaxSize) { - opts.MaxRepoSize = opts.MaxSize - } - client := *opts.Client - client.Timeout = 0 - client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } - - uploadPacks := uploadPacksPerCPU * runtime.GOMAXPROCS(0) - if opts.maxUploadPacks > 0 { - uploadPacks = opts.maxUploadPacks - } - refListings := refListingsPerCPU * runtime.GOMAXPROCS(0) - if opts.maxRefListings > 0 { - refListings = opts.maxRefListings - } - c := &Cache{ - opts: opts, - client: &client, - logger: logger, - gitPath: gitPath, - dir: dir, - lock: lock, - fetches: make(chan struct{}, maxConcurrentFetches), - uploadPacks: make(chan struct{}, uploadPacks), - refListings: make(chan struct{}, refListings), - refCheckTimeout: defaultRefCheckTimeout, - lowSpeedTime: defaultLowSpeedTime, - maxRequestBody: defaultMaxRequestBody, - repos: make(map[string]*repo), - tooLarge: make(map[string]time.Time), - failedWants: make(map[string]time.Time), - freed: make(chan struct{}), - foldCase: caseInsensitive(dir), - } - if opts.refCheckTimeout > 0 { - c.refCheckTimeout = opts.refCheckTimeout - } - if opts.lowSpeedTime > 0 { - c.lowSpeedTime = opts.lowSpeedTime - } - if opts.maxRequestBody > 0 { - c.maxRequestBody = opts.maxRequestBody - } - if err := c.load(); err != nil { - _ = lock.Close() - return nil, err - } - if c.relay, err = newRelay(c.client, logger); err != nil { - _ = lock.Close() - return nil, fmt.Errorf("starting git relay: %w", err) - } - c.ctx, c.cancel = context.WithCancel(context.Background()) - // Applies a lowered git.max_size straight away. - c.evict() - return c, nil -} - -// Close stops the refreshes in progress and waits for them to finish, so -// none is still writing to the cache when it returns, then stops the relay. -// Fetches by ID run on request goroutines; stop the HTTP server first. -// Closing again does nothing. -func (c *Cache) Close() error { - c.mu.Lock() - if c.closed { - c.mu.Unlock() - return nil - } - c.closed = true - c.mu.Unlock() - c.cancel() - c.refreshes.Wait() - return errors.Join(c.relay.close(), c.lock.Close()) -} - -// UpstreamDir names the top-level cache directory of the upstream called -// name at baseURL. It includes a hash of the URL, so repointing an upstream -// starts a fresh cache rather than serving the old host's repositories. -func UpstreamDir(name, baseURL string) string { - digest := sha256.Sum256([]byte(strings.TrimSuffix(baseURL, "/"))) - return name + "-" + hex.EncodeToString(digest[:])[:upstreamHashLen] -} - -func (c *Cache) tmpDir() string { - return filepath.Join(c.dir, tmpDirName) -} - -// load clears what an earlier run left in the temporary directory and -// registers the repositories already cached. -func (c *Cache) load() error { - if err := os.RemoveAll(c.tmpDir()); err != nil { - return fmt.Errorf("clearing %s: %w", c.tmpDir(), err) - } - if err := os.MkdirAll(c.tmpDir(), dirPermissions); err != nil { - return err - } - c.removeRepointed() - for _, upstreamDir := range c.opts.Upstreams { - root := filepath.Join(c.dir, upstreamDir) - err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - if err != nil || !d.IsDir() || !strings.HasSuffix(d.Name(), ".git") || path == root { - return err - } - rel, _ := filepath.Rel(c.dir, path) - rp := newRepo(c.repoKey(filepath.ToSlash(rel)), path) - rp.size.Store(dirSize(path)) - rp.lastChecked.Store(stampTime(path, checkedStamp)) - rp.lastUsed.Store(stampTime(path, usedStamp)) - rp.usedWritten.Store(rp.lastUsed.Load()) - c.repos[rp.key] = rp - return filepath.SkipDir - }) - if err != nil && !errors.Is(err, fs.ErrNotExist) { - return err - } - } - return nil -} - -// upstreamDirPattern matches a directory UpstreamDir names, capturing the -// upstream's name. -var upstreamDirPattern = regexp.MustCompile(`^(.+)-[0-9a-f]{8}$`) - -// removeRepointed removes the directories of configured upstreams at URLs -// they no longer have. Each is moved aside first, so nothing can be read -// from it half deleted. Directories of upstreams no longer configured are -// only logged: nothing else says they are the cache's. -func (c *Cache) removeRepointed() { - entries, err := os.ReadDir(c.dir) - if err != nil { - c.logger.Warn("git cache: cannot list the cache directory", "error", err) - return - } - current := make(map[string]bool, len(c.opts.Upstreams)) - for _, upstreamDir := range c.opts.Upstreams { - current[upstreamDir] = true - } - var unconfigured []string - for _, entry := range entries { - name := entry.Name() - match := upstreamDirPattern.FindStringSubmatch(name) - if !entry.IsDir() || match == nil || current[name] { - continue - } - if _, configured := c.opts.Upstreams[match[1]]; !configured { - unconfigured = append(unconfigured, name) - continue - } - trash, err := os.MkdirTemp(c.tmpDir(), "repointed-") - if err == nil { - err = os.Rename(filepath.Join(c.dir, name), filepath.Join(trash, name)) - } - if err != nil { - c.logger.Warn("git cache: failed to remove the cache of a repointed upstream", "directory", name, "error", err) - continue - } - c.logger.Info("git cache: removed the cache of a repointed upstream", "directory", name) - if err := os.RemoveAll(trash); err != nil { - c.logger.Warn("git cache: failed to delete a removed cache", "path", trash, "error", err) - } - } - if len(unconfigured) > 0 { - c.logger.Info("git cache: leaving directories that look like caches of upstreams no longer configured; delete them if they are", - "directories", unconfigured) - } -} - -// acquire returns the repository for key, counting the caller as a user -// until release, so it is not evicted while in use. -func (c *Cache) acquire(key string) *repo { - c.mu.Lock() - defer c.mu.Unlock() - rk := c.repoKey(key) - rp := c.repos[rk] - if rp == nil { - rp = newRepo(rk, filepath.Join(c.dir, filepath.FromSlash(key))) - c.repos[rk] = rp - } - rp.users++ - return rp -} - -// repoKey is the key a repository is tracked under: key itself, or key -// lowercased where the cache directory ignores case. -func (c *Cache) repoKey(key string) string { - if c.foldCase { - return strings.ToLower(key) - } - return key -} - -func (c *Cache) release(rp *repo) { - now := time.Now() - rp.lastUsed.Store(now.UnixNano()) - exists := rp.exists() - if exists && now.Sub(time.Unix(0, rp.usedWritten.Load())) >= usedStampInterval { - rp.usedWritten.Store(now.UnixNano()) - touch(rp.dir, usedStamp, now) - } - - c.mu.Lock() - rp.users-- - idle := rp.users == 0 && c.repos[rp.key] == rp - // Forget repositories that never got cloned, such as ones missing upstream. - if idle && !exists { - delete(c.repos, rp.key) - } - _, tooLarge := c.tooLarge[rp.key] - retry := tooLarge || c.overBudget - c.mu.Unlock() - if idle && exists && retry { - c.evict() - } -} - -func newRepo(key, dir string) *repo { - return &repo{key: key, dir: dir, writes: make(chan struct{}, 1)} -} - -// lockWrites waits for other fetches into the repository to finish. -func (rp *repo) lockWrites(ctx context.Context) (func(), error) { - select { - case rp.writes <- struct{}{}: - return func() { <-rp.writes }, nil - case <-ctx.Done(): - return nil, ctx.Err() - } -} - -func (rp *repo) exists() bool { - _, err := os.Stat(rp.dir) - return err == nil -} - -// evict removes repositories nobody is using: the least recently used until -// the cache fits in MaxSize, and any that could not be refreshed within -// MaxRepoSize, so the next request clones them again with only their -// branches and tags. It runs after every fetch, and when a repository it had -// to skip is released: one too large to refresh, or any while the cache is -// over MaxSize. Fetches also evict as they receive bytes (reservation.cover). -func (c *Cache) evict() { - c.mu.Lock() - removed, _ := c.evictLocked(0) - c.mu.Unlock() - c.removeEvicted(removed) -} - -// evictLocked evicts until the cache, counting reservations, has room for -// extra more bytes, and returns the evicted repositories' directories, to be -// deleted once c.mu is released, and the size left. When evicting every idle -// repository would not make that room, it evicts only as evict does. A -// repository is moved aside under the lock, so a request that comes after it -// clones it again instead of reading it while it is deleted. -func (c *Cache) evictLocked(extra int64) ([]string, int64) { - total := c.reserved - var idleTotal int64 - idle := make([]*repo, 0, len(c.repos)) - for _, rp := range c.repos { - total += rp.size.Load() - if rp.users == 0 { - idle = append(idle, rp) - idleTotal += rp.size.Load() - } - } - if total-idleTotal+extra > c.opts.MaxSize { - extra = 0 - } - slices.SortFunc(idle, func(a, b *repo) int { - return cmp.Compare(a.lastUsed.Load(), b.lastUsed.Load()) - }) - var removed []string - for _, rp := range idle { - _, marked := c.tooLarge[rp.key] - tooLarge := marked && rp.exists() - if !tooLarge && (c.opts.MaxSize <= 0 || total+extra <= c.opts.MaxSize) { - continue - } - trash, err := os.MkdirTemp(c.tmpDir(), "evicted-") - if err == nil { - err = os.Rename(rp.dir, filepath.Join(trash, "repo")) - } - if err != nil && !errors.Is(err, fs.ErrNotExist) { - c.logger.Warn("git cache: failed to evict repository", "repository", rp.key, "error", err) - continue - } - delete(c.repos, rp.key) - total -= rp.size.Load() - removed = append(removed, trash) - reason := "lru" - if tooLarge { - // The clone that replaces it enforces the limit itself. - delete(c.tooLarge, rp.key) - reason = "too_large" - } - metrics.RecordArtifactEvicted(reason, ecosystem) - c.logger.Info("git cache: evicted repository", "repository", rp.key, "size", rp.size.Load(), "reason", reason) - } - c.overBudget = c.opts.MaxSize > 0 && total > c.opts.MaxSize - return removed, total -} - -func (c *Cache) removeEvicted(removed []string) { - for _, trash := range removed { - if err := os.RemoveAll(trash); err != nil { - c.logger.Warn("git cache: failed to delete evicted repository", "path", trash, "error", err) - } - } -} - -func dirSize(dir string) int64 { - var size int64 - _ = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { - if err == nil && d.Type().IsRegular() { - if info, err := d.Info(); err == nil { - size += info.Size() - } - } - return nil - }) - return size -} - -// Stamps are empty files whose modification time records an event, so it -// survives restarts. -func stampTime(dir, name string) int64 { - info, err := os.Stat(filepath.Join(dir, name)) - if err != nil { - return 0 - } - return info.ModTime().UnixNano() -} - -func touch(dir, name string, t time.Time) { - path := filepath.Join(dir, name) - if err := os.Chtimes(path, t, t); errors.Is(err, fs.ErrNotExist) { - if f, err := os.Create(path); err == nil { - _ = f.Close() - _ = os.Chtimes(path, t, t) - } - } -} - -// caseInsensitive reports whether the file system holding dir ignores case -// in names, as macOS and Windows do by default. -func caseInsensitive(dir string) bool { - f, err := os.CreateTemp(dir, ".gitcache-case-") - if err != nil { - return false - } - name := f.Name() - _ = f.Close() - defer func() { _ = os.Remove(name) }() - _, err = os.Stat(filepath.Join(dir, strings.ToUpper(filepath.Base(name)))) - return err == nil -} diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go deleted file mode 100644 index 2b2e388..0000000 --- a/internal/gitcache/cache_test.go +++ /dev/null @@ -1,1034 +0,0 @@ -package gitcache - -import ( - "context" - "crypto/rand" - "errors" - "fmt" - "maps" - "math" - "net/http" - "net/http/httptest" - "net/http/httputil" - "net/url" - "os" - "path/filepath" - "slices" - "strings" - "sync" - "testing" - "time" - - "github.com/git-pkgs/registries/safehttp" -) - -func TestCloneMatchesUpstream(t *testing.T) { - for _, version := range []string{"0", "1", "2"} { - t.Run("protocol v"+version, func(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - - clone := tc.clone(t, "repo.git", "-c", "protocol.version="+version) - - want := runGit(t, "", "ls-remote", "--heads", "--tags", upstream.bare()) - if got := runGit(t, "", "-c", "protocol.version="+version, "ls-remote", "--heads", "--tags", tc.url("repo.git")); got != want { - t.Errorf("ls-remote through the cache:\n%s\nwant:\n%s", got, want) - } - if got, want := runGit(t, clone, "rev-parse", "HEAD"), runGit(t, upstream.bare(), "rev-parse", "main"); got != want { - t.Errorf("HEAD = %s, want %s", got, want) - } - runGit(t, clone, "fsck", "--no-dangling", "--no-progress") - }) - } -} - -func TestUpstreamWithoutProtocolV2(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.v0Only = true - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - - clone := tc.clone(t, "repo") - if got, want := runGit(t, clone, "rev-parse", "HEAD"), runGit(t, upstream.bare(), "rev-parse", "main"); got != want { - t.Errorf("HEAD = %s, want %s", got, want) - } -} - -func TestUnchangedRepositoryIsNotFetchedAgain(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - - tc.clone(t, "repo") - fetches := upstream.gitRequests.Load() - tc.clone(t, "repo") - if got := upstream.gitRequests.Load(); got != fetches { - t.Errorf("second clone made %d upstream fetch requests, want none", got-fetches) - } -} - -func TestPushedCommitIsServedOnNextClone(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - existing := tc.clone(t, "repo") - - head := upstream.commit(work, "second") - runGit(t, work, "push", "--quiet", "origin", "main") - - if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != head { - t.Errorf("clone after push: HEAD = %s, want %s", got, head) - } - runGit(t, existing, "pull", "--quiet", "--ff-only") - if got := runGit(t, existing, "rev-parse", "HEAD"); got != head { - t.Errorf("pull after push: HEAD = %s, want %s", got, head) - } - if got := runGit(t, tc.clone(t, "repo", "--depth=1"), "rev-list", "--count", "HEAD"); got != "1" { - t.Errorf("shallow clone has %s commits, want 1", got) - } -} - -func TestOnlyBranchesAndTagsAreCachedAndPruned(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, work, "push", "--quiet", "origin", "main:refs/changes/01/1/1", "main:refs/heads/gone", "main:refs/tags/gone") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - - refs := tc.cachedRefs(t) - if strings.Contains(refs, "refs/changes/") { - t.Errorf("cached refs/changes:\n%s", refs) - } - if !strings.Contains(refs, "refs/heads/gone") || !strings.Contains(refs, "refs/tags/gone") { - t.Fatalf("branch and tag not cached:\n%s", refs) - } - - runGit(t, work, "push", "--quiet", "origin", ":refs/heads/gone", ":refs/tags/gone") - tc.clone(t, "repo") - if refs := tc.cachedRefs(t); strings.Contains(refs, "gone") { - t.Errorf("deleted branch or tag still cached:\n%s", refs) - } - - // Gerrit advertises a ref per patch set, so asking for every ref would - // list hundreds of thousands for a large repository. - bodies := upstream.lsRefsBodies() - if len(bodies) == 0 { - t.Fatal("no ls-refs request reached the upstream") - } - for _, body := range bodies { - if !strings.Contains(body, "ref-prefix refs/heads/") || !strings.Contains(body, "ref-prefix refs/tags/") { - t.Errorf("ls-refs asked for every ref, want branches and tags only:\n%q", body) - } - } -} - -func TestDefaultBranchChangeIsFollowed(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - - runGit(t, work, "checkout", "--quiet", "-b", "trunk") - head := upstream.commit(work, "on trunk") - runGit(t, work, "push", "--quiet", "origin", "trunk") - runGit(t, upstream.bare(), "symbolic-ref", "HEAD", "refs/heads/trunk") - - clone := tc.clone(t, "repo") - if got := runGit(t, clone, "rev-parse", "HEAD"); got != head { - t.Errorf("HEAD = %s, want trunk's %s", got, head) - } -} - -func TestConcurrentColdClonesShareOneFetch(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - - release := upstream.holdRequests() - var wg sync.WaitGroup - for range 8 { - wg.Go(func() { - dir := filepath.Join(t.TempDir(), "clone") - if out, err := tryGit(t, "", "clone", "--quiet", tc.url("repo"), dir); err != nil { - t.Errorf("clone: %v\n%s", err, out) - } - }) - } - time.Sleep(200 * time.Millisecond) - release() - wg.Wait() - - if got := upstream.gitSessions.Load(); got != 1 { - t.Errorf("upstream saw %d fetches, want 1", got) - } -} - -func TestMissingRepositoryLeavesNothingBehind(t *testing.T) { - upstream := newFakeUpstream(t) - tc := newTestCache(t, upstream.server.URL, nil) - - resp, err := http.Get(tc.url("missing") + "/info/refs?service=git-upload-pack") - if err != nil { - t.Fatal(err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusNotFound { - t.Errorf("status %d, want 404", resp.StatusCode) - } - waitForRepos(t, tc.Cache, 0) - if _, err := os.Stat(filepath.Join(tc.dir, "up", "missing.git")); !os.IsNotExist(err) { - t.Errorf("repository directory left behind: %v", err) - } -} - -func TestStaleCopyServedOnlyWithinStaleMaxAge(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - - upstream.status.Store(http.StatusServiceUnavailable) - tc.clone(t, "repo") - - waitIdle(t, tc.Cache) - rp := tc.acquire("up/repo.git") - rp.lastChecked.Store(time.Now().Add(-2 * time.Hour).UnixNano()) - tc.release(rp) - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Errorf("served a copy older than stale_max_age:\n%s", out) - } -} - -func TestRefusedRefreshIsNotServedStale(t *testing.T) { - for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, http.StatusMovedPermanently} { - t.Run(fmt.Sprint(status), func(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - - upstream.status.Store(int32(status)) - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Errorf("served the cached copy after status %d:\n%s", status, out) - } - }) - } -} - -func TestSlowUpstreamServesCachedCopyAfterRefreshWait(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = 100 * time.Millisecond }) - old := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD") - head := upstream.commit(work, "second") - runGit(t, work, "push", "--quiet", "origin", "main") - - release := upstream.holdRequests() - defer release() - if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != old { - t.Errorf("HEAD = %s, want the cached %s", got, old) - } - release() - waitFor(t, func() bool { return strings.Contains(tc.cachedRefs(t), head) }) -} - -// TestSlowFetchAfterPushIsWaitedFor has the upstream answer the ref check at -// once and the fetch after a push still run when RefreshWait is over: a -// clone still gets the pushed commit. -func TestSlowFetchAfterPushIsWaitedFor(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - waitOver := make(chan time.Time) - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.refreshWaitOver = waitOver }) - tc.clone(t, "repo") - head := upstream.commit(work, "second") - runGit(t, work, "push", "--quiet", "origin", "main") - - arrived, release := upstream.holdFetches() - defer release() - dir := filepath.Join(t.TempDir(), "clone") - cloned := make(chan error, 1) - go func() { - out, err := tryGit(t, "", "clone", "--quiet", tc.url("repo"), dir) - if err != nil { - err = fmt.Errorf("%w\n%s", err, out) - } - cloned <- err - }() - // The fetch starts only after the check found the push. - select { - case <-arrived: - case err := <-cloned: - t.Fatalf("clone finished before the fetch started: %v", err) - case <-time.After(30 * time.Second): - t.Fatal("the fetch after the push never reached the upstream") - } - close(waitOver) - release() - if err := <-cloned; err != nil { - t.Fatal(err) - } - if got := runGit(t, dir, "rev-parse", "HEAD"); got != head { - t.Errorf("HEAD = %s, want the pushed %s", got, head) - } -} - -// TestStalledFetchAfterPushServesCachedCopy has the fetch after a push -// stall: once it is abandoned, the clone gets the cached copy. -func TestStalledFetchAfterPushServesCachedCopy(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { - o.RefreshWait = 100 * time.Millisecond - o.lowSpeedTime = 1 - }) - old := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD") - upstream.commit(work, "second") - runGit(t, work, "push", "--quiet", "origin", "main") - - upstream.stallFetches.Store(true) - if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != old { - t.Errorf("HEAD = %s, want the cached %s", got, old) - } -} - -func TestUpstreamCredentialsReachEveryUpstreamRequest(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.requireAuth = "Bearer secret" - upstream.newRepo("repo") - client := &http.Client{Transport: headerTransport{"Authorization", "Bearer secret"}} - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.Client = client }) - - tc.clone(t, "repo") - if got := upstream.unauthed.Load(); got != 0 { - t.Errorf("%d upstream requests lacked credentials", got) - } - if upstream.gitRequests.Load() == 0 { - t.Error("no fetch went through the relay") - } -} - -type headerTransport struct{ name, value string } - -func (h headerTransport) RoundTrip(r *http.Request) (*http.Response, error) { - r = r.Clone(r.Context()) - r.Header.Set(h.name, h.value) - return http.DefaultTransport.RoundTrip(r) -} - -// TestUpstreamAddressChecksApplyToGit uses a client that refuses loopback -// addresses, as the proxy's does by default. git must not reach the loopback -// upstream around it, whether the ref check or only git's fetch is refused. -func TestUpstreamAddressChecksApplyToGit(t *testing.T) { - blocked := safehttp.New(nil, safehttp.Options{}) - for name, client := range map[string]*http.Client{ - "every request": blocked, - "git's fetch": {Transport: gitOnlyTransport{blocked.Transport}}, - } { - t.Run(name, func(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.Client = client }) - - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Fatalf("clone through a blocked upstream succeeded:\n%s", out) - } - if got := upstream.gitRequests.Load(); got != 0 { - t.Errorf("git reached the upstream with %d requests", got) - } - }) - } -} - -// gitOnlyTransport sends the cache's own ref checks with the default -// transport, and git's relayed requests with next. -type gitOnlyTransport struct{ next http.RoundTripper } - -func (g gitOnlyTransport) RoundTrip(r *http.Request) (*http.Response, error) { - if r.UserAgent() == gitUserAgent { - return http.DefaultTransport.RoundTrip(r) - } - return g.next.RoundTrip(r) -} - -// TestRedirectIsNotFollowed redirects the ref check, then only git's fetch. -func TestRedirectIsNotFollowed(t *testing.T) { - for _, redirectRefCheck := range []bool{true, false} { - t.Run(fmt.Sprint("ref check redirected: ", redirectRefCheck), func(t *testing.T) { - target := newFakeUpstream(t) - target.newRepo("repo") - targetURL, _ := url.Parse(target.server.URL) - passThrough := httputil.NewSingleHostReverseProxy(targetURL) - redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.UserAgent() == gitUserAgent && !redirectRefCheck { - passThrough.ServeHTTP(w, r) - return - } - http.Redirect(w, r, target.server.URL+r.URL.RequestURI(), http.StatusFound) - })) - defer redirect.Close() - tc := newTestCache(t, redirect.URL, nil) - - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Fatalf("followed a redirect:\n%s", out) - } - if got := target.gitRequests.Load(); got != 0 { - t.Errorf("redirect target received %d requests from git", got) - } - }) - } -} - -func TestRepositoryOverMaxRepoSizeIsRefused(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - big := make([]byte, 256<<10) - _, _ = rand.Read(big) - if err := os.WriteFile(filepath.Join(work, "big.bin"), big, 0o600); err != nil { - t.Fatal(err) - } - runGit(t, work, "add", "big.bin") - runGit(t, work, "commit", "--quiet", "-m", "big") - runGit(t, work, "push", "--quiet", "origin", "main") - - // max_size caps max_repo_size. - for name, configure := range map[string]func(*Options){ - "max_repo_size": func(o *Options) { o.MaxRepoSize = 64 << 10 }, - "max_size": func(o *Options) { o.MaxSize = 64 << 10 }, - } { - t.Run(name, func(t *testing.T) { - tc := newTestCache(t, upstream.server.URL, configure) - - resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") - if err != nil { - t.Fatal(err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusInsufficientStorage { - t.Errorf("status %d, want 507", resp.StatusCode) - } - waitForRepos(t, tc.Cache, 0) - if entries, _ := os.ReadDir(tc.tmpDir()); len(entries) != 0 { - t.Errorf("left %d entries in the temporary directory", len(entries)) - } - - requests := upstream.requests.Load() - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Errorf("second attempt succeeded:\n%s", out) - } - if got := upstream.requests.Load(); got != requests { - t.Errorf("second attempt made %d upstream requests, want none", got-requests) - } - }) - } -} - -// TestEvictionRemovesLeastRecentlyUsedIdleRepositories makes the least -// recently used repository one a request is reading, so the next one is -// evicted instead. -func TestEvictionRemovesLeastRecentlyUsedIdleRepositories(t *testing.T) { - upstream := newFakeUpstream(t) - for _, name := range []string{"old", "busy", "new"} { - upstream.newRepo(name) - } - size := repoSize(t, upstream, "new") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = 2*size + size/2 }) - - tc.clone(t, "busy") - busy := tc.acquire("up/busy.git") - defer tc.release(busy) - busy.lastUsed.Store(0) - tc.clone(t, "old") - waitFor(t, func() bool { return tc.usersOf("up/old.git") == 0 }) - tc.repo("up/old.git").lastUsed.Store(1) - - // The third clone takes the cache past MaxSize. - tc.clone(t, "new") - waitFor(t, func() bool { return !fileExists(filepath.Join(tc.dir, "up", "old.git")) }) - for _, name := range []string{"busy", "new"} { - if !fileExists(filepath.Join(tc.dir, "up", name+".git")) { - t.Errorf("%s was evicted", name) - } - } - tc.clone(t, "old") -} - -// repoSize is the size a cache gives repository name once cloned. -func repoSize(t *testing.T, upstream *fakeUpstream, name string) int64 { - t.Helper() - probe := newTestCache(t, upstream.server.URL, nil) - probe.clone(t, name) - return probe.repo("up/" + name + ".git").size.Load() -} - -func TestNewLoadsCachedRepositories(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - if err := os.MkdirAll(filepath.Join(tc.tmpDir(), "clone-leftover"), 0o750); err != nil { - t.Fatal(err) - } - if err := tc.Close(); err != nil { - t.Fatal(err) - } - - reloaded, err := New(Options{ - Upstreams: map[string]string{"up": "up"}, - CacheDir: tc.dir, - StaleMaxAge: time.Hour, - Client: &http.Client{}, - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = reloaded.Close() }() - - rp := reloaded.repos["up/repo.git"] // nothing else uses reloaded yet - if rp == nil || rp.size.Load() == 0 || rp.lastChecked.Load() == 0 { - t.Fatalf("repository not loaded: %+v", rp) - } - if entries, _ := os.ReadDir(reloaded.tmpDir()); len(entries) != 0 { - t.Errorf("temporary directory not cleared: %v", entries) - } -} - -func waitForRepos(t *testing.T, c *Cache, n int) { - t.Helper() - waitFor(t, func() bool { - c.mu.Lock() - defer c.mu.Unlock() - return len(c.repos) == n - }) -} - -func waitFor(t *testing.T, cond func() bool) { - t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - for !cond() { - select { - case <-ctx.Done(): - t.Fatal("condition not met in time") - case <-time.After(20 * time.Millisecond): - } - } -} - -// TestRewoundCacheIsBroughtBackInLine changes the cache behind the upstream's -// back, so it only matches again if the next request refreshes branches and -// tags and prunes what the upstream does not have. -func TestRewoundCacheIsBroughtBackInLine(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - upstream.commit(work, "second") - runGit(t, work, "push", "--quiet", "origin", "main") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - - cached := filepath.Join(tc.dir, "up", "repo.git") - runGit(t, cached, "update-ref", "refs/heads/main", "main~1") - runGit(t, cached, "update-ref", "refs/heads/extra", "main") - runGit(t, cached, "tag", "extra", "main") - - want := runGit(t, "", "ls-remote", "--heads", "--tags", upstream.bare()) - if got := runGit(t, "", "ls-remote", "--heads", "--tags", tc.url("repo")); got != want { - t.Errorf("ls-remote through the cache:\n%s\nwant:\n%s", got, want) - } -} - -// TestConcurrentWarmClonesDuringRefresh serves clones from the cache while a -// fetch writes to it. -func TestConcurrentWarmClonesDuringRefresh(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = time.Millisecond }) - old := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD") - head := upstream.commit(work, "second") - runGit(t, work, "push", "--quiet", "origin", "main") - - var wg sync.WaitGroup - for range 8 { - wg.Go(func() { - dir := filepath.Join(t.TempDir(), "clone") - if out, err := tryGit(t, "", "clone", "--quiet", tc.url("repo"), dir); err != nil { - t.Errorf("clone: %v\n%s", err, out) - return - } - if got := runGit(t, dir, "rev-parse", "HEAD"); got != old && got != head { - t.Errorf("HEAD = %s, want %s or %s", got, old, head) - } - runGit(t, dir, "fsck", "--no-dangling", "--no-progress") - }) - } - wg.Wait() - waitFor(t, func() bool { return strings.Contains(tc.cachedRefs(t), head) }) -} - -// TestStalledRefCheckFailsInsteadOfHanging has the upstream send headers and -// then nothing, as a broken load balancer might. -func TestStalledRefCheckFailsInsteadOfHanging(t *testing.T) { - stall := make(chan struct{}) - defer close(stall) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", advertisementType) - w.WriteHeader(http.StatusOK) - _ = http.NewResponseController(w).Flush() - select { - case <-stall: - case <-r.Context().Done(): - } - })) - defer upstream.Close() - tc := newTestCache(t, upstream.URL, func(o *Options) { o.refCheckTimeout = 200 * time.Millisecond }) - - start := time.Now() - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Fatalf("ls-remote succeeded:\n%s", out) - } - if elapsed := time.Since(start); elapsed > 10*time.Second { - t.Errorf("took %s to fail", elapsed) - } -} - -// TestStalledFetchFailsInsteadOfHanging has the upstream answer the ref check -// but never git's fetch, as a connection that stalls mid-pack would. -func TestStalledFetchFailsInsteadOfHanging(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - upstream.stallFetches.Store(true) - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.lowSpeedTime = 1 }) - - start := time.Now() - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Fatalf("ls-remote succeeded:\n%s", out) - } - if elapsed := time.Since(start); elapsed > 20*time.Second { - t.Errorf("took %s to fail", elapsed) - } -} - -// TestFetchEvictsToMaxSize checks eviction runs on its own once a fetch has -// grown the cache past MaxSize. -func TestFetchEvictsToMaxSize(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("first") - upstream.newRepo("second") - size := repoSize(t, upstream, "first") - - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = size + size/2 }) - tc.clone(t, "first") - waitIdle(t, tc.Cache) - tc.clone(t, "second") - waitFor(t, func() bool { return !fileExists(filepath.Join(tc.dir, "up", "first.git")) }) - if !fileExists(filepath.Join(tc.dir, "up", "second.git")) { - t.Error("evicted the repository just fetched") - } -} - -// TestReleaseEvictsWhatEvictionHadToSkip fills the cache past MaxSize while -// every repository is in use, so eviction after the fetch can remove none. -// The first release brings the cache back within MaxSize. -func TestReleaseEvictsWhatEvictionHadToSkip(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("first") - upstream.newRepo("second") - size := repoSize(t, upstream, "first") - - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = size + size/2 }) - first := tc.acquire("up/first.git") - tc.clone(t, "first") - second := tc.acquire("up/second.git") - defer tc.release(second) - tc.clone(t, "second") - waitFor(t, func() bool { - tc.mu.Lock() - defer tc.mu.Unlock() - return tc.overBudget - }) - - tc.release(first) - if fileExists(first.dir) { - t.Error("first is still cached once released") - } - if !fileExists(second.dir) { - t.Error("evicted second while in use") - } - tc.clone(t, "second") -} - -// TestNewRemovesRepointedUpstreams starts a cache whose directory holds an -// upstream's cache at its old URL, and one of an upstream no longer -// configured. Only the first is known to be the cache's to remove. -func TestNewRemovesRepointedUpstreams(t *testing.T) { - dir := t.TempDir() - current := UpstreamDir("up", "https://new.example") - repointed := UpstreamDir("up", "https://old.example") - unconfigured := UpstreamDir("gone", "https://gone.example") - // Named after a configured upstream's name, but another upstream's. - longer := UpstreamDir("up-api", "https://api.example") - for _, name := range []string{current, repointed, unconfigured, longer} { - runGit(t, "", "init", "--quiet", "--bare", filepath.Join(dir, name, "repo.git")) - } - c, err := New(Options{ - CacheDir: dir, - StaleMaxAge: time.Hour, - Client: &http.Client{}, - Upstreams: map[string]string{"up": current}, - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = c.Close() }() - - for name, want := range map[string]bool{current: true, repointed: false, unconfigured: true, longer: true} { - if got := fileExists(filepath.Join(dir, name)); got != want { - t.Errorf("%s present = %v, want %v", name, got, want) - } - } - if keys := slices.Collect(maps.Keys(c.repos)); len(keys) != 1 || keys[0] != current+"/repo.git" { - t.Errorf("loaded %v, want only %s/repo.git", keys, current) - } -} - -// TestCacheDirSharedWithOtherData points the cache at a directory that also -// holds an artifact store and a database, as git.cache_dir: ./cache would. -func TestCacheDirSharedWithOtherData(t *testing.T) { - dir := t.TempDir() - _ = os.MkdirAll(filepath.Join(dir, "artifacts", "npm"), 0o750) - _ = os.WriteFile(filepath.Join(dir, "artifacts", "npm", "pkg.tgz"), []byte("x"), 0o600) - _ = os.WriteFile(filepath.Join(dir, "proxy.db"), []byte("db"), 0o600) - runGit(t, "", "init", "--quiet", "--bare", filepath.Join(dir, "artifacts", "foo", "bar.git")) - c, err := New(Options{ - CacheDir: dir, - Client: &http.Client{}, - Upstreams: map[string]string{"github": UpstreamDir("github", "https://github.com")}, - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = c.Close() }() - for _, p := range []string{"artifacts/npm/pkg.tgz", "proxy.db", "artifacts/foo/bar.git"} { - if _, err := os.Stat(filepath.Join(dir, p)); err != nil { - t.Errorf("%s was deleted at startup: %v", p, err) - } - } - if len(c.repos) != 0 { - t.Errorf("loaded %v from outside the upstream directories", slices.Collect(maps.Keys(c.repos))) - } -} - -// TestCloseWaitsForRefreshes closes the cache while a clone is held up -// upstream. When Close returns, the clone must have finished, temporary -// directory included, so nothing is left writing to the cache. -func TestCloseWaitsForRefreshes(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - upstream.stallFetches.Store(true) - tc := newTestCache(t, upstream.server.URL, nil) - clones := func() []string { - names, _ := filepath.Glob(filepath.Join(tc.tmpDir(), "clone-*")) - return names - } - - go func() { _, _ = tryGit(t, "", "ls-remote", tc.url("repo")) }() - waitFor(t, func() bool { return len(clones()) > 0 }) - if err := tc.Close(); err != nil { - t.Fatal(err) - } - if left := clones(); len(left) != 0 { - t.Errorf("Close returned with the clone still running: %v", left) - } - - before := upstream.requests.Load() - if _, err := tc.Open(context.Background(), "up/other.git", upstream.server.URL+"/other", OpenOptions{}); !errors.Is(err, errClosed) { - t.Errorf("Open after Close: %v, want errClosed", err) - } - if got := upstream.requests.Load() - before; got != 0 { - t.Errorf("Open after Close made %d upstream requests", got) - } -} - -// TestCacheDirInUse opens a second cache on a directory one already uses, -// as a second pod on a shared volume would, then again once the first has -// closed. -func TestCacheDirInUse(t *testing.T) { - dir := t.TempDir() - open := func() (*Cache, error) { return New(Options{CacheDir: dir, Client: &http.Client{}}) } - first, err := open() - if err != nil { - t.Fatal(err) - } - if second, err := open(); err == nil { - _ = second.Close() - t.Fatal("a second cache opened on the same directory") - } else if !strings.Contains(err.Error(), "in use by another process") { - t.Errorf("error = %v, want one naming the directory as in use", err) - } - if err := first.Close(); err != nil { - t.Fatal(err) - } - again, err := open() - if err != nil { - t.Fatalf("reopening after Close: %v", err) - } - _ = again.Close() -} - -// TestCloseTwice checks closing again does nothing, as a cleanup that also -// closes the cache may. -func TestCloseTwice(t *testing.T) { - c, err := New(Options{CacheDir: t.TempDir(), Client: &http.Client{}}) - if err != nil { - t.Fatal(err) - } - if err := errors.Join(c.Close(), c.Close()); err != nil { - t.Errorf("closing twice: %v", err) - } -} - -// lsRefs sends a protocol v2 ls-refs request for "repo" straight to -// git-upload-pack, as a client may without asking for info/refs first, and -// returns the status. -func (tc *testCache) lsRefs(t *testing.T) int { - t.Helper() - body := pktLine("command=ls-refs\n") + pktDelim + pktLine("peel\n") + pktFlush - req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, tc.url("repo")+"/git-upload-pack", strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } - req.Header.Set("Content-Type", requestType) - req.Header.Set("Git-Protocol", "version=2") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } - _ = resp.Body.Close() - return resp.StatusCode -} - -// TestUploadPackChecksCopyPastStaleMaxAge sends upload-pack requests without -// info/refs: a copy not checked within stale_max_age is checked first. -func TestUploadPackChecksCopyPastStaleMaxAge(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - if status := tc.lsRefs(t); status != http.StatusOK { - t.Fatalf("ls-refs on a fresh copy: status %d", status) - } - - waitIdle(t, tc.Cache) - rp := tc.acquire("up/repo.git") - rp.lastChecked.Store(time.Now().Add(-2 * time.Hour).UnixNano()) - tc.release(rp) - upstream.status.Store(http.StatusNotFound) - if status := tc.lsRefs(t); status == http.StatusOK { - t.Errorf("served a copy older than stale_max_age to ls-refs") - } -} - -// TestUploadPackRefusedAfterUpstreamRefusal has the upstream refuse a -// refresh: upload-pack requests are refused too until a check succeeds. -func TestUploadPackRefusedAfterUpstreamRefusal(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - - upstream.status.Store(http.StatusForbidden) - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Fatalf("served the cached copy after 403:\n%s", out) - } - if status := tc.lsRefs(t); status == http.StatusOK { - t.Errorf("ls-refs served after the upstream refused the repository") - } - - upstream.status.Store(0) - tc.clone(t, "repo") - if status := tc.lsRefs(t); status != http.StatusOK { - t.Errorf("ls-refs after a successful check: status %d", status) - } -} - -// TestCloneChecksUpstreamOnce clones with a stale_max_age of 0: the -// upload-pack requests after info/refs do not check the upstream again. -func TestCloneChecksUpstreamOnce(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.StaleMaxAge = 0 }) - tc.clone(t, "repo") - waitIdle(t, tc.Cache) - - checks := func() int32 { return upstream.requests.Load() - upstream.gitRequests.Load() } - before := checks() - resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") - if err != nil { - t.Fatal(err) - } - _ = resp.Body.Close() - perCheck := checks() - before - - before = checks() - tc.clone(t, "repo") - if got := checks() - before; got != perCheck { - t.Errorf("clone made %d upstream check requests, want %d", got, perCheck) - } -} - -// TestInfoRefsRefusedWhileRefusalStands checks that once the upstream -// refused a repository, a later check that runs past RefreshWait, which -// would otherwise serve the cached copy, doesn't advertise its refs. -func TestInfoRefsRefusedWhileRefusalStands(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = 100 * time.Millisecond }) - tc.clone(t, "repo") - - upstream.status.Store(http.StatusForbidden) - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Fatalf("served the cached copy after 403:\n%s", out) - } - upstream.status.Store(0) - release := upstream.holdRequests() - defer release() - // Protocol v0 advertises the refs in the info/refs response itself. - if out, err := tryGit(t, "", "-c", "protocol.version=0", "ls-remote", tc.url("repo")); err == nil { - t.Errorf("advertised refs while the refusal stands:\n%s", out) - } -} - -// refuse has the upstream refuse "repo" with a 403 and checks that the -// cached copy is no longer served. -func (tc *testCache) refuse(t *testing.T, upstream *fakeUpstream) { - t.Helper() - upstream.status.Store(http.StatusForbidden) - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Fatalf("served the cached copy after 403:\n%s", out) - } - upstream.status.Store(0) -} - -// TestUploadPackClearsRefusal has a client that only sends upload-pack -// requests: once the upstream allows the repository again, it is served. -func TestUploadPackClearsRefusal(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - tc.refuse(t, upstream) - - if status := tc.lsRefs(t); status != http.StatusOK { - t.Errorf("ls-refs after the upstream allowed the repository again: status %d", status) - } -} - -// TestRefCheckClearsRefusal has the upstream list the refs again after a -// refusal but fail the fetch that follows: the check alone clears the -// refusal, and the cached copy is served. -func TestRefCheckClearsRefusal(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - tc.refuse(t, upstream) - - upstream.commit(work, "second") - runGit(t, work, "push", "--quiet", "origin", "main") - upstream.fetchStatus.Store(http.StatusInternalServerError) - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err != nil { - t.Errorf("cached copy refused after the upstream listed its refs: %v\n%s", err, out) - } - if status := tc.lsRefs(t); status != http.StatusOK { - t.Errorf("ls-refs after the upstream listed its refs: status %d", status) - } -} - -// TestHugeStaleMaxAgeUploadPackChecksNothing sets stale_max_age near the -// largest duration: upload-pack requests on a fresh copy do not check the -// upstream. -func TestHugeStaleMaxAgeUploadPackChecksNothing(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.StaleMaxAge = math.MaxInt64 - time.Second }) - tc.clone(t, "repo") - waitIdle(t, tc.Cache) - - before := upstream.requests.Load() - if status := tc.lsRefs(t); status != http.StatusOK { - t.Fatalf("ls-refs: status %d", status) - } - waitIdle(t, tc.Cache) - if got := upstream.requests.Load() - before; got != 0 { - t.Errorf("ls-refs made %d upstream requests, want 0", got) - } -} - -// TestFetchRedirectRefusesRepository has the upstream list the refs but -// redirect the fetch that follows: the repository is refused, as when the -// ref listing is redirected. -func TestFetchRedirectRefusesRepository(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - - upstream.commit(work, "second") - runGit(t, work, "push", "--quiet", "origin", "main") - upstream.fetchStatus.Store(http.StatusFound) - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Errorf("served the cached copy after the fetch was redirected:\n%s", out) - } - if status := tc.lsRefs(t); status == http.StatusOK { - t.Errorf("ls-refs served after the fetch was redirected") - } -} - -// TestCaseOnlyDifferentKeysShareOneRepository checks that, where the file -// system ignores case, keys that differ only in case are one repository, so -// a user of one spelling keeps the directory from being evicted under it. -func TestCaseOnlyDifferentKeysShareOneRepository(t *testing.T) { - if !caseInsensitive(t.TempDir()) { - t.Skip("file system is case-sensitive") - } - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - waitIdle(t, tc.Cache) - - rp := tc.acquire("up/REPO.git") - defer tc.release(rp) - if other := tc.repo("up/repo.git"); other != rp { - t.Error("up/REPO.git and up/repo.git are separate repositories") - } - tc.mu.Lock() - tc.opts.MaxSize = 1 - tc.mu.Unlock() - tc.evict() - if !rp.exists() { - t.Error("repository in use as up/REPO.git was evicted") - } -} - -// TestFoldCaseKeysShareOneEntry checks key folding on any file system by -// setting foldCase directly. -func TestFoldCaseKeysShareOneEntry(t *testing.T) { - tc := newTestCache(t, newFakeUpstream(t).server.URL, nil) - tc.foldCase = true - - upper := tc.acquire("up/REPO.git") - defer tc.release(upper) - lower := tc.acquire("up/repo.git") - defer tc.release(lower) - if upper != lower { - t.Error("up/REPO.git and up/repo.git are separate repositories") - } - tc.mu.Lock() - n := len(tc.repos) - tc.mu.Unlock() - if n != 1 { - t.Errorf("%d repository entries, want 1", n) - } -} diff --git a/internal/gitcache/git.go b/internal/gitcache/git.go deleted file mode 100644 index 0cc2c11..0000000 --- a/internal/gitcache/git.go +++ /dev/null @@ -1,104 +0,0 @@ -package gitcache - -import ( - "bytes" - "context" - "fmt" - "os" - "os/exec" - "slices" - "strconv" - "strings" - "time" -) - -const ( - maxStderr = 4 << 10 - gitWaitDelay = 5 * time.Second -) - -// gitConfig applies to every git process the cache runs. -var gitConfig = [][2]string{ - // git would follow a redirect from the relay straight to the network. - {"http.followRedirects", "false"}, - {"protocol.version", "2"}, - // Auto gc stays on, as on any Git server, but runs inside the fetch - // rather than detaching, so no git process outlives the fetch that - // started it, and sizes are measured after it. - {"gc.autoDetach", "false"}, - {"maintenance.autoDetach", "false"}, - {"init.defaultBranch", "main"}, - // Keep every fetched pack as a pack. index-pack writes it to a - // temporary file and moves it into place only once complete, whereas a - // small pack is otherwise unpacked object by object, and a transfer cut - // off at max_repo_size would leave the objects before the cut, such as a - // commit without its tree's blobs. - {"transfer.unpackLimit", "1"}, - // Serve any commit the cache holds when a client asks for it by object - // ID, as the upstream would, including ones a force-push left behind, - // and do not advertise the refs that keep fetched ones. - {"uploadpack.allowAnySHA1InWant", "true"}, - {"uploadpack.hideRefs", wantedPrefix}, - // With http.lowSpeedTime, added per cache, gives up on a transfer that - // receives nothing for that long, so a connection that stalls mid-pack - // cannot hold a refresh for fetchTimeout. - {"http.lowSpeedLimit", "1"}, - // index-pack streams blobs larger than this instead of holding them in - // memory, and pack-objects stores them without delta compression, so a - // large file costs a fetch or a clone little memory. - {"core.bigFileThreshold", "8m"}, -} - -// command builds a git process with an environment of its own: none of the -// proxy's environment is passed on, so no system or user config, credential -// helper, HTTP proxy setting or GIT_ variable applies, and the only transport -// git may use is plain HTTP, which is what the relay speaks. -func (c *Cache) command(ctx context.Context, extraEnv []string, args ...string) *exec.Cmd { - cmd := exec.CommandContext(ctx, c.gitPath, args...) - env := []string{ - "PATH=" + os.Getenv("PATH"), - "HOME=" + c.tmpDir(), - "LC_ALL=C", - "GIT_CONFIG_NOSYSTEM=1", - "GIT_CONFIG_GLOBAL=" + os.DevNull, - "GIT_TERMINAL_PROMPT=0", - "GIT_ALLOW_PROTOCOL=http", - } - config := slices.Concat(gitConfig, [][2]string{{"http.lowSpeedTime", strconv.Itoa(c.lowSpeedTime)}}) - env = append(env, "GIT_CONFIG_COUNT="+strconv.Itoa(len(config))) - for i, kv := range config { - env = append(env, - "GIT_CONFIG_KEY_"+strconv.Itoa(i)+"="+kv[0], - "GIT_CONFIG_VALUE_"+strconv.Itoa(i)+"="+kv[1]) - } - env = append(env, extraEnv...) - cmd.Env = env - cmd.WaitDelay = gitWaitDelay - return cmd -} - -// run runs git in the repository at dir and returns its standard output. -func (c *Cache) run(ctx context.Context, dir string, args ...string) (string, error) { - cmd := c.command(ctx, nil, append([]string{"-C", dir}, args...)...) - var stdout bytes.Buffer - stderr := &limitedBuffer{max: maxStderr} - cmd.Stdout = &stdout - cmd.Stderr = stderr - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("git %s: %w: %s", args[0], err, strings.TrimSpace(stderr.String())) - } - return stdout.String(), nil -} - -// limitedBuffer keeps the first max bytes written to it. -type limitedBuffer struct { - bytes.Buffer - max int -} - -func (b *limitedBuffer) Write(p []byte) (int, error) { - if room := b.max - b.Len(); room > 0 { - b.Buffer.Write(p[:min(len(p), room)]) - } - return len(p), nil -} diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go deleted file mode 100644 index 8e6c2e5..0000000 --- a/internal/gitcache/helpers_test.go +++ /dev/null @@ -1,437 +0,0 @@ -package gitcache - -import ( - "bytes" - "compress/gzip" - "errors" - "io" - "log" - "log/slog" - "net/http" - "net/http/cgi" - "net/http/httptest" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "testing" - "time" -) - -// gitEnv isolates the tests' own git commands from the machine's config. -func gitEnv(home string) []string { - return []string{ - "PATH=" + os.Getenv("PATH"), - "HOME=" + home, - "GIT_CONFIG_NOSYSTEM=1", - "GIT_TERMINAL_PROMPT=0", - "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", - "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", - } -} - -func runGit(t *testing.T, dir string, args ...string) string { - t.Helper() - out, err := tryGit(t, dir, args...) - if err != nil { - t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) - } - return strings.TrimSpace(out) -} - -func tryGit(t *testing.T, dir string, args ...string) (string, error) { - t.Helper() - cmd := exec.Command("git", args...) - cmd.Dir = dir - cmd.Env = gitEnv(t.TempDir()) - out, err := cmd.CombinedOutput() - return string(out), err -} - -// httpBackendPath finds git http-backend, which serves the fake upstreams, -// and skips the test where git is installed without it. -func httpBackendPath(t *testing.T) string { - t.Helper() - dir := runGit(t, "", "--exec-path") - for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { - if path := filepath.Join(dir, name); fileExists(path) { - return path - } - } - t.Skip("git http-backend is not installed") - return "" -} - -func fileExists(path string) bool { - _, err := os.Stat(path) - return err == nil -} - -// fakeUpstream serves bare repositories under root with git http-backend. -type fakeUpstream struct { - t *testing.T - root string - server *httptest.Server - - // status, when set, is returned instead of serving the request, and - // fetchStatus instead of serving git's own upload-pack requests. - status atomic.Int32 - fetchStatus atomic.Int32 - // requireAuth, when set, is the Authorization every request must carry. - requireAuth string - // v0Only drops Git-Protocol, so only protocol v0 is spoken. - v0Only bool - // hold, when set, blocks requests until it is closed. - holdMu sync.Mutex - hold chan struct{} - - requests atomic.Int32 - // gitRequests counts requests from git processes, which reach the - // upstream through the relay, and gitSessions the fetches they make. - gitRequests atomic.Int32 - gitSessions atomic.Int32 - unauthed atomic.Int32 - - // stallFetches makes git's own upload-pack requests hang, and - // delayFetches holds them for that long. - stallFetches atomic.Bool - delayFetches atomic.Int64 - // fetchHold, while set, holds git's own upload-pack requests until it - // is closed; fetchArrived is closed when the first one arrives. - fetchHold chan struct{} - fetchArrived chan struct{} - fetchOnce *sync.Once - - bodiesMu sync.Mutex - bodies []string -} - -// recordBody keeps the decoded body of a POST. It also gives the request a -// Content-Length, since net/http/cgi refuses chunked bodies, which git sends -// once a request outgrows http.postBuffer. -func (u *fakeUpstream) recordBody(r *http.Request) { - raw, err := io.ReadAll(r.Body) - if err != nil { - u.t.Errorf("reading request body: %v", err) - } - r.Body = io.NopCloser(bytes.NewReader(raw)) - r.ContentLength = int64(len(raw)) - r.TransferEncoding = nil - decoded := raw - if r.Header.Get("Content-Encoding") == "gzip" { - if zr, err := gzip.NewReader(bytes.NewReader(raw)); err == nil { - decoded, _ = io.ReadAll(zr) - } - } - u.bodiesMu.Lock() - u.bodies = append(u.bodies, string(decoded)) - u.bodiesMu.Unlock() -} - -// lsRefsBodies returns the ls-refs requests the upstream received. -func (u *fakeUpstream) lsRefsBodies() []string { - u.bodiesMu.Lock() - defer u.bodiesMu.Unlock() - var out []string - for _, body := range u.bodies { - if strings.Contains(body, "command=ls-refs") { - out = append(out, body) - } - } - return out -} - -func newFakeUpstream(t *testing.T) *fakeUpstream { - t.Helper() - u := &fakeUpstream{t: t, root: t.TempDir()} - backend := &cgi.Handler{ - Path: httpBackendPath(t), - Stderr: io.Discard, - Logger: log.New(io.Discard, "", 0), - Env: append(gitEnv(t.TempDir()), - "GIT_PROJECT_ROOT="+u.root, - "GIT_HTTP_EXPORT_ALL=1"), - } - u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - u.requests.Add(1) - if r.UserAgent() != gitUserAgent { - u.gitRequests.Add(1) - if r.Method == http.MethodGet { - u.gitSessions.Add(1) - } - } - u.holdMu.Lock() - hold := u.hold - u.holdMu.Unlock() - if hold != nil { - <-hold - } - if status := u.status.Load(); status != 0 { - http.Error(w, "unavailable", int(status)) - return - } - if u.requireAuth != "" && r.Header.Get("Authorization") != u.requireAuth { - u.unauthed.Add(1) - http.Error(w, "unauthorized", http.StatusUnauthorized) - return - } - if u.v0Only { - r.Header.Del("Git-Protocol") - } - if r.UserAgent() != gitUserAgent && r.Method == http.MethodPost && u.interceptFetch(w, r) { - return - } - if r.Method == http.MethodPost { - u.recordBody(r) - } - backend.ServeHTTP(w, r) - })) - t.Cleanup(u.server.Close) - return u -} - -// interceptFetch applies the fetch knobs to one of git's own upload-pack -// requests, and reports whether it answered the request. -func (u *fakeUpstream) interceptFetch(w http.ResponseWriter, r *http.Request) bool { - u.holdMu.Lock() - fetchHold, fetchArrived, fetchOnce := u.fetchHold, u.fetchArrived, u.fetchOnce - u.holdMu.Unlock() - if fetchHold != nil { - fetchOnce.Do(func() { close(fetchArrived) }) - <-fetchHold - } - if delay := u.delayFetches.Load(); delay > 0 { - time.Sleep(time.Duration(delay)) - } - if u.stallFetches.Load() { - // The server notices the client leaving only once the body is read. - _, _ = io.Copy(io.Discard, r.Body) - <-r.Context().Done() - return true - } - if status := u.fetchStatus.Load(); status != 0 { - w.Header().Set("Location", u.server.URL+"/elsewhere") - http.Error(w, "unavailable", int(status)) - return true - } - return false -} - -// holdFetches holds git's own upload-pack requests until release is called. -// arrived is closed when the first one reaches the upstream. -func (u *fakeUpstream) holdFetches() (arrived <-chan struct{}, release func()) { - u.holdMu.Lock() - defer u.holdMu.Unlock() - hold, first := make(chan struct{}), make(chan struct{}) - u.fetchHold, u.fetchArrived, u.fetchOnce = hold, first, &sync.Once{} - var once sync.Once - return first, func() { - once.Do(func() { - u.holdMu.Lock() - u.fetchHold = nil - u.holdMu.Unlock() - close(hold) - }) - } -} - -func (u *fakeUpstream) holdRequests() func() { - u.holdMu.Lock() - defer u.holdMu.Unlock() - hold := make(chan struct{}) - u.hold = hold - var once sync.Once - return func() { - once.Do(func() { - u.holdMu.Lock() - u.hold = nil - u.holdMu.Unlock() - close(hold) - }) - } -} - -// newRepo creates {name}.git upstream with one commit on main and a tag, and -// returns a working clone to commit and push from. -func (u *fakeUpstream) newRepo(name string) string { - u.t.Helper() - bare := filepath.Join(u.root, name+".git") - runGit(u.t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) - work := u.t.TempDir() - runGit(u.t, "", "clone", "--quiet", bare, work) - runGit(u.t, work, "checkout", "--quiet", "-b", "main") - u.commit(work, "first") - runGit(u.t, work, "tag", "-a", "-m", "v1", "v1") - runGit(u.t, work, "push", "--quiet", "origin", "main", "v1") - return work -} - -func (u *fakeUpstream) commit(work, msg string) string { - u.t.Helper() - if err := os.WriteFile(filepath.Join(work, "file.txt"), []byte(msg+"\n"), 0o600); err != nil { - u.t.Fatal(err) - } - runGit(u.t, work, "add", "file.txt") - runGit(u.t, work, "commit", "--quiet", "-m", msg) - return runGit(u.t, work, "rev-parse", "HEAD") -} - -// waitIdle waits until no request or refresh is using a repository, so a -// test can inspect or change the cache without racing a request finishing. -func waitIdle(t *testing.T, c *Cache) { - t.Helper() - waitFor(t, func() bool { - c.mu.Lock() - defer c.mu.Unlock() - for _, rp := range c.repos { - if rp.users > 0 { - return false - } - } - return true - }) -} - -// bare is the upstream repository named "repo". -// repo returns the cache's entry for key, reading the map under the lock. -func (tc *testCache) repo(key string) *repo { - tc.mu.Lock() - defer tc.mu.Unlock() - return tc.repos[key] -} - -// usersOf is the number of requests and refreshes using the repository. -func (tc *testCache) usersOf(key string) int { - tc.mu.Lock() - defer tc.mu.Unlock() - return tc.repos[key].users -} - -func (u *fakeUpstream) bare() string { - return filepath.Join(u.root, "repo.git") -} - -type testCache struct { - *Cache - server *httptest.Server - logs *lockedBuffer -} - -func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *testCache { - t.Helper() - // The cache's log is shown when the test fails. - logs := &lockedBuffer{} - t.Cleanup(func() { - if t.Failed() { - t.Logf("git cache log:\n%s", logs.String()) - } - }) - opts := Options{ - Upstreams: map[string]string{"up": "up"}, - CacheDir: t.TempDir(), - RefreshWait: 5 * time.Second, - StaleMaxAge: time.Hour, - Client: &http.Client{}, - Logger: slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})), - } - if configure != nil { - configure(&opts) - } - c, err := New(opts) - if err != nil { - t.Fatalf("New: %v", err) - } - server := httptest.NewServer(testFront(c, upstreamURL, true)) - t.Cleanup(func() { - server.Close() - _ = c.Close() - }) - return &testCache{Cache: c, server: server, logs: logs} -} - -// testFront serves /git/up/{repo}/info/refs and git-upload-pack from c, as -// internal/handler does once it has validated the request, with repo -// fetched from upstreamURL/{repo}. fetchWants is OpenOptions.FetchWants. -func testFront(c *Cache, upstreamURL string, fetchWants bool) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - path := strings.TrimPrefix(r.URL.Path, "/git/up/") - repo, infoRefs := strings.CutSuffix(path, "/info/refs") - if !infoRefs { - var ok bool - if repo, ok = strings.CutSuffix(path, "/git-upload-pack"); !ok || r.Method != http.MethodPost { - http.Error(w, "not served", http.StatusForbidden) - return - } - } else if r.URL.RawQuery != "service=git-upload-pack" { - http.Error(w, "not served", http.StatusForbidden) - return - } - var req *Request - if !infoRefs { - var err error - if req, err = c.ReadRequest(r); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - defer req.Close() - } - lease, err := c.Open(r.Context(), "up/"+strings.TrimSuffix(repo, ".git")+".git", upstreamURL+"/"+repo, - OpenOptions{Refresh: infoRefs, FetchWants: fetchWants}) - switch { - case errors.Is(err, ErrNotFound): - http.Error(w, err.Error(), http.StatusNotFound) - return - case errors.Is(err, ErrTooLarge), errors.Is(err, ErrCacheFull): - http.Error(w, err.Error(), http.StatusInsufficientStorage) - return - case err != nil: - http.Error(w, err.Error(), http.StatusBadGateway) - return - } - defer lease.Release() - if infoRefs { - lease.ServeInfoRefs(w, r) - } else { - lease.ServeUploadPack(w, r, req) - } - }) -} - -func (tc *testCache) url(repo string) string { - return tc.server.URL + "/git/up/" + repo -} - -// clone clones repo through the cache with the given protocol version and -// returns the working copy. -func (tc *testCache) clone(t *testing.T, repo string, args ...string) string { - t.Helper() - dir := filepath.Join(t.TempDir(), "clone") - runGit(t, "", append(append([]string{"clone", "--quiet"}, args...), tc.url(repo), dir)...) - return dir -} - -// cachedRefs lists the refs of the cached copy of "repo". -func (tc *testCache) cachedRefs(t *testing.T) string { - t.Helper() - return runGit(t, filepath.Join(tc.dir, "up", "repo.git"), "for-each-ref", "--format=%(objectname) %(refname)") -} - -// lockedBuffer is a bytes.Buffer safe for concurrent writers. -type lockedBuffer struct { - mu sync.Mutex - buf bytes.Buffer -} - -func (b *lockedBuffer) Write(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.Write(p) -} - -func (b *lockedBuffer) String() string { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.String() -} diff --git a/internal/gitcache/lock_unix.go b/internal/gitcache/lock_unix.go deleted file mode 100644 index 215027a..0000000 --- a/internal/gitcache/lock_unix.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build !windows - -package gitcache - -import ( - "errors" - "os" - "syscall" -) - -func lockFile(path string) (*os.File, error) { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, lockFilePermissions) - if err != nil { - return nil, err - } - if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - _ = f.Close() - if errors.Is(err, syscall.EWOULDBLOCK) { - return nil, errLocked - } - return nil, err - } - return f, nil -} diff --git a/internal/gitcache/lock_windows.go b/internal/gitcache/lock_windows.go deleted file mode 100644 index 99f46d5..0000000 --- a/internal/gitcache/lock_windows.go +++ /dev/null @@ -1,27 +0,0 @@ -package gitcache - -import ( - "errors" - "os" - "syscall" -) - -const errorSharingViolation syscall.Errno = 32 - -// lockFile opens path with no sharing, so no other handle can open it while -// this one is open. -func lockFile(path string) (*os.File, error) { - name, err := syscall.UTF16PtrFromString(path) - if err != nil { - return nil, err - } - h, err := syscall.CreateFile(name, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, - syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) - if errors.Is(err, errorSharingViolation) { - return nil, errLocked - } - if err != nil { - return nil, err - } - return os.NewFile(uintptr(h), path), nil -} diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go deleted file mode 100644 index 6e45d43..0000000 --- a/internal/gitcache/mirror.go +++ /dev/null @@ -1,253 +0,0 @@ -package gitcache - -import ( - "context" - "errors" - "fmt" - "os/exec" - "slices" - "strings" - "time" -) - -// MirrorResult is what Mirror did. -type MirrorResult struct { - // Cached reports that the repository, and the commit if a version was - // given, were cached already. - Cached bool - // Bytes is how much the cached repository grew. - Bytes int64 - // Commit is the commit the version names, empty without a version. - Commit string - // Submodules are the submodules Commit records, sorted by path. - Submodules []Submodule -} - -// Submodule is a submodule a commit records: its path, the URL .gitmodules -// gives for it, which may be relative to the repository's, and the commit -// the gitlink pins. -type Submodule struct { - Path, URL, Commit string -} - -// Mirror brings the repository cached under key up to date with repoURL, -// cloning it if needed, and waits for that to finish. version, if given, is -// a full commit ID or a tag. A commit is made sure to be cached: one on no -// branch or tag is fetched by ID when fetchWants allows it (see -// OpenOptions.FetchWants), and is kept as long as a commit a client fetched -// by ID. A tag is cached with the repository's tags. -func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) { - if lower := strings.ToLower(version); isObjectID(lower) { - version = lower - } - tag := version != "" && !isObjectID(version) - if tag { - if err := c.checkTagName(ctx, version); err != nil { - return MirrorResult{}, err - } - } - lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants, NoCheck: true}) - if err != nil { - return MirrorResult{}, err - } - defer lease.Release() - rp := lease.repo - cached := lease.hit - before := int64(0) - var refreshErr error - // Open cloned a repository that wasn't cached; one that was is - // refreshed here, waiting for the fetch a client would not wait for. - if cached { - before = rp.size.Load() - refresh := c.startRefresh(rp, repoURL) - select { - case <-refresh.done: - refreshErr = refresh.err - case <-ctx.Done(): - return MirrorResult{}, ctx.Err() - } - } - - // A refresh past max_repo_size gets the repository dropped once it is - // released, commit and all. - if errors.Is(refreshErr, ErrTooLarge) { - return MirrorResult{}, refreshErr - } - commit := version - switch { - case tag: - if commit, err = c.resolveTag(ctx, rp.dir, version); err != nil { - if refreshErr != nil { - // The tag may be one the failed refresh would have brought. - return MirrorResult{}, refreshErr - } - if errors.Is(err, ErrNotFound) && strings.Trim(strings.ToLower(version), "0123456789abcdef") == "" { - // Likely an abbreviated commit ID. - return MirrorResult{}, fmt.Errorf("%s is not a full commit ID or tag: %w", version, ErrNotFound) - } - return MirrorResult{}, err - } - if refreshErr != nil { - // Tags can move; this is where it pointed when last fetched. - c.logger.Warn("git cache: refresh failed, mirroring the cached tag", "repository", key, "tag", version, "commit", commit, "error", refreshErr) - } - case version != "": - if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { - return MirrorResult{}, err - } - } - // A refresh that failed leaves the commit cached all the same. - if refreshErr != nil && version == "" { - return MirrorResult{}, refreshErr - } - var submodules []Submodule - if commit != "" { - if submodules, err = c.submodules(ctx, key, rp.dir, commit); err != nil { - return MirrorResult{}, err - } - } - grown := max(rp.size.Load()-before, 0) - return MirrorResult{Cached: cached && grown == 0, Bytes: grown, Commit: commit, Submodules: submodules}, nil -} - -// checkTagName refuses a version that is no valid tag name, such as one -// with .. or a revision suffix like ^{tree}. -func (c *Cache) checkTagName(ctx context.Context, name string) error { - if err := c.command(ctx, nil, "check-ref-format", "refs/tags/"+name).Run(); err != nil { - return fmt.Errorf("%q is neither a full commit ID nor a tag name", name) - } - return nil -} - -// resolveTag returns the commit tag points at. -func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) { - out, err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", "refs/tags/"+tag+"^{commit}").Output() - var exit *exec.ExitError - if errors.As(err, &exit) && exit.ExitCode() == 1 { - return "", fmt.Errorf("tag %s: %w", tag, ErrNotFound) - } - if err != nil { - return "", fmt.Errorf("resolving tag %s: %w", tag, err) - } - return strings.TrimSpace(string(out)), nil -} - -// submodules lists the submodules commit records: its gitlinks that -// .gitmodules names a URL for. -func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Submodule, error) { - err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", commit+":.gitmodules").Run() - var exit *exec.ExitError - if errors.As(err, &exit) && exit.ExitCode() == 1 && ctx.Err() == nil { - return nil, nil - } - if err != nil { - if ctx.Err() != nil { - return nil, ctx.Err() - } - return nil, fmt.Errorf("looking for .gitmodules: %w", err) - } - // --list rather than --get-regexp, which fails when nothing matches. - out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list") - if err != nil { - if ctx.Err() != nil { - return nil, ctx.Err() - } - // The commit is cached; a .gitmodules git can't read only loses - // its submodules. - c.logger.Warn("git cache: cannot read .gitmodules, skipping submodules", "repository", key, "commit", commit, "error", err) - return nil, nil - } - paths, urls := map[string]string{}, map[string]string{} - for entry := range strings.SplitSeq(out, "\x00") { - key, value, ok := strings.Cut(entry, "\n") - section, ok2 := strings.CutPrefix(key, "submodule.") - name, field, dotted := strings.CutLast(section, ".") - if !ok || !ok2 || !dotted { - continue - } - switch field { - case "path": - paths[value] = name - case "url": - urls[name] = value - } - } - var pathspecs []string - for path, name := range paths { - if path != "" && urls[name] != "" { - pathspecs = append(pathspecs, path) - } - } - if len(pathspecs) == 0 { - return nil, nil - } - args := append([]string{"--literal-pathspecs", "ls-tree", "-z", "--full-tree", commit, "--"}, pathspecs...) - tree, err := c.run(ctx, dir, args...) - if err != nil { - return nil, err - } - var submodules []Submodule - for entry := range strings.SplitSeq(tree, "\x00") { - meta, path, ok := strings.Cut(entry, "\t") - fields := strings.Fields(meta) - if !ok || len(fields) != 3 || fields[0] != "160000" { - continue - } - if url := urls[paths[path]]; url != "" && isObjectID(fields[2]) { - submodules = append(submodules, Submodule{Path: path, URL: url, Commit: fields[2]}) - } - } - slices.SortFunc(submodules, func(a, b Submodule) int { return strings.Compare(a.Path, b.Path) }) - return submodules, nil -} - -// ensureCommit makes sure the repository has commit, fetching it by ID when -// it doesn't and fetchWants allows it. A commit on no branch or tag that is -// cached already gets a new wanted ref, so it is kept for wantedMaxAge from -// this run on. -func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) error { - missing, err := c.missingObjects(ctx, rp.dir, []string{commit}) - if err != nil { - return err - } - if len(missing) == 0 { - return c.renewWanted(ctx, rp, commit) - } - if !fetchWants { - return fmt.Errorf("commit %s: %w", commit, ErrNotFound) - } - if err := c.fetchMissingWants(ctx, rp, repoURL, missing); err != nil { - if ctx.Err() != nil || errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) { - return err - } - // The upstream refuses a commit it doesn't have. - return fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err) - } - // fetchMissingWants skips a commit that failed moments ago. - if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil { - return err - } - if len(missing) > 0 { - return fmt.Errorf("commit %s: %w", commit, ErrNotFound) - } - return nil -} - -// renewWanted writes a new wanted ref for commit when no branch or tag -// includes it. -func (c *Cache) renewWanted(ctx context.Context, rp *repo, commit string) error { - unlock, err := rp.lockWrites(ctx) - if err != nil { - return err - } - defer unlock() - unmerged, err := c.unmergedCommits(ctx, rp.dir, []string{commit}) - if err != nil || !unmerged[commit] { - return err - } - cmd := c.command(ctx, nil, "-C", rp.dir, "update-ref", wantedRef(commit, time.Now()), commit) - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("renewing wanted ref: %w: %s", err, strings.TrimSpace(string(out))) - } - return nil -} diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go deleted file mode 100644 index d016f2a..0000000 --- a/internal/gitcache/mirror_test.go +++ /dev/null @@ -1,415 +0,0 @@ -package gitcache - -import ( - "context" - "errors" - "net/http" - "os" - "path/filepath" - "slices" - "strconv" - "strings" - "testing" - "time" -) - -func TestMirrorClonesRepository(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - - got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) - if err != nil { - t.Fatalf("Mirror: %v", err) - } - if got.Cached || got.Bytes <= 0 { - t.Errorf("first mirror = %+v, want fetched bytes", got) - } - if !strings.Contains(tc.cachedRefs(t), "refs/tags/v1") { - t.Errorf("cached refs %q lack the tag", tc.cachedRefs(t)) - } - - got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) - if err != nil || !got.Cached { - t.Errorf("second mirror = %+v, %v, want cached", got, err) - } -} - -func TestMirrorFetchesCommitOnNoBranch(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - sha := upstream.pushPullRequest(work, 1, nil) - tc := newTestCache(t, upstream.server.URL, nil) - - got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) - if err != nil { - t.Fatalf("Mirror: %v", err) - } - if got.Cached { - t.Errorf("mirror = %+v, want fetched", got) - } - if refs := tc.wantedRefs(t); len(refs) != 1 { - t.Errorf("wanted refs %v, want one", refs) - } - - sessions := upstream.gitSessions.Load() - got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) - if err != nil || !got.Cached { - t.Errorf("second mirror = %+v, %v, want cached", got, err) - } - if n := upstream.gitSessions.Load() - sessions; n != 0 { - t.Errorf("second mirror fetched %d times, want none", n) - } -} - -func TestMirrorFindsCommitOnABranch(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - sha := upstream.commit(work, "second") - runGit(t, work, "push", "--quiet", "origin", "main") - tc := newTestCache(t, upstream.server.URL, nil) - - got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, false) - if err != nil || got.Cached { - t.Errorf("mirror = %+v, %v, want fetched", got, err) - } -} - -func TestMirrorDoesNotFetchByIDWhenTurnedOff(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - sha := upstream.pushPullRequest(work, 1, nil) - tc := newTestCache(t, upstream.server.URL, nil) - - _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, false) - if !errors.Is(err, ErrNotFound) { - t.Errorf("mirror of a commit on no branch with fetching by ID off = %v, want ErrNotFound", err) - } - if refs := tc.wantedRefs(t); len(refs) != 0 { - t.Errorf("wanted refs %v, want none", refs) - } -} - -func TestMirrorReportsMissingCommit(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - - _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", strings.Repeat("ab", 20), true) - if !errors.Is(err, ErrNotFound) { - t.Errorf("mirror of a commit the upstream lacks = %v, want ErrNotFound", err) - } -} - -func TestMirrorRefusesInvalidCommit(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - - if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "main", true); err == nil { - t.Error("mirrored a branch name as a commit") - } -} - -func TestMirrorCountsRefreshThatFetched(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true); err != nil { - t.Fatalf("Mirror: %v", err) - } - upstream.commit(work, "second") - runGit(t, work, "push", "--quiet", "origin", "main") - - got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) - if err != nil { - t.Fatalf("Mirror: %v", err) - } - if got.Cached || got.Bytes <= 0 { - t.Errorf("mirror after a push = %+v, want fetched bytes", got) - } -} - -// TestMirrorRenewsWantedCommit checks mirroring a commit on no branch again -// makes its 7 days count from that run, not from the first fetch. -func TestMirrorRenewsWantedCommit(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - sha := upstream.pushPullRequest(work, 1, nil) - tc := newTestCache(t, upstream.server.URL, nil) - if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true); err != nil { - t.Fatalf("Mirror: %v", err) - } - cached := filepath.Join(tc.dir, "up", "repo.git") - for _, ref := range tc.wantedRefs(t) { - runGit(t, cached, "update-ref", "-d", ref) - } - old := wantedRef(sha, time.Now().Add(-wantedMaxAge+time.Hour)) - runGit(t, cached, "update-ref", old, sha) - - start := time.Now().Add(-time.Second) - got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) - if err != nil || !got.Cached { - t.Fatalf("second mirror = %+v, %v, want cached", got, err) - } - if !slices.ContainsFunc(tc.wantedRefs(t), func(ref string) bool { - return ref != old && wantedRefTime(t, ref, sha).After(start) - }) { - t.Errorf("wanted refs %v, want one written by the second mirror", tc.wantedRefs(t)) - } -} - -func TestMirrorDoesNotPinCommitOnABranch(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - sha := upstream.commit(work, "second") - runGit(t, work, "push", "--quiet", "origin", "main") - tc := newTestCache(t, upstream.server.URL, nil) - for range 2 { - if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true); err != nil { - t.Fatalf("Mirror: %v", err) - } - } - if refs := tc.wantedRefs(t); len(refs) != 0 { - t.Errorf("wanted refs %v, want none for a commit on a branch", refs) - } -} - -func wantedRefTime(t *testing.T, ref, oid string) time.Time { - t.Helper() - created, got, _ := strings.Cut(strings.TrimPrefix(ref, wantedPrefix), "/") - unix, err := strconv.ParseInt(created, 10, 64) - if err != nil || got != oid { - return time.Time{} - } - return time.Unix(unix, 0) -} - -// TestMirrorReportsRefreshTooLarge checks a commit that is cached still -// fails to mirror when the refresh doesn't fit in max_repo_size, since the -// repository is dropped once released. -func TestMirrorReportsRefreshTooLarge(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - pinned := upstream.pushPullRequest(work, 1, randomBytes(100<<10)) - base := repoSize(t, upstream, "repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 150<<10 }) - if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", pinned, true); err != nil { - t.Fatalf("Mirror: %v", err) - } - if err := os.WriteFile(filepath.Join(work, "grown.bin"), randomBytes(80<<10), 0o600); err != nil { - t.Fatal(err) - } - runGit(t, work, "add", "grown.bin") - upstream.commit(work, "grown") - runGit(t, work, "push", "--quiet", "origin", "main") - - _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", pinned, true) - if !errors.Is(err, ErrTooLarge) { - t.Errorf("mirror with a refresh past max_repo_size = %v, want ErrTooLarge", err) - } -} - -func TestMirrorResolvesTag(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - tagged := runGit(t, work, "rev-parse", "v1^{commit}") - tc := newTestCache(t, upstream.server.URL, nil) - - got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true) - if err != nil { - t.Fatalf("Mirror: %v", err) - } - if got.Commit != tagged { - t.Errorf("commit %q, want %s the tag points at", got.Commit, tagged) - } -} - -func TestMirrorReportsMissingTag(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - - for _, version := range []string{"v9", "main"} { - _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) - if !errors.Is(err, ErrNotFound) { - t.Errorf("mirror at %q = %v, want ErrNotFound", version, err) - } - } -} - -func TestMirrorRefusesInvalidTag(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - - for _, version := range []string{"a..b", "v1^{tree}", "v1~1", "v 1", "v1:x"} { - _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) - if err == nil || errors.Is(err, ErrNotFound) { - t.Errorf("mirror at %q = %v, want an invalid version error", version, err) - } - } -} - -func TestMirrorListsSubmodules(t *testing.T) { - upstream := newFakeUpstream(t) - libWork := upstream.newRepo("lib") - libCommit := runGit(t, libWork, "rev-parse", "HEAD") - work := upstream.newRepo("repo") - gitmodules := "[submodule \"vendor/lib\"]\n\tpath = vendor/lib\n\turl = ../lib.git\n" + - "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + - "[submodule \"colon\"]\n\tpath = :colon\n\turl = ../lib.git\n" - if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { - t.Fatal(err) - } - otherCommit := strings.Repeat("ab", 20) - runGit(t, work, "add", ".gitmodules") - runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") - runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") - // Git for Windows refuses ':' in paths unless NTFS protection is off. - runGit(t, work, "-c", "core.protectNTFS=false", "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",:colon") - runGit(t, work, "commit", "--quiet", "-m", "submodules") - runGit(t, work, "push", "--quiet", "origin", "main") - commit := runGit(t, work, "rev-parse", "HEAD") - tc := newTestCache(t, upstream.server.URL, nil) - - got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) - if err != nil { - t.Fatalf("Mirror: %v", err) - } - want := []Submodule{ - {Path: ":colon", URL: "../lib.git", Commit: libCommit}, - {Path: "third_party/other", URL: "https://example.com/other.git", Commit: otherCommit}, - {Path: "vendor/lib", URL: "../lib.git", Commit: libCommit}, - } - if !slices.Equal(got.Submodules, want) { - t.Errorf("submodules %+v, want %+v", got.Submodules, want) - } - - // Without a version, nothing names a commit to read submodules from. - got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) - if err != nil || len(got.Submodules) != 0 { - t.Errorf("unversioned mirror = %+v, %v, want no submodules", got, err) - } -} - -// TestMirrorWithEmptyGitmodules checks a .gitmodules with no path or url, -// for which git config --get-regexp finds nothing, lists no submodules. -func TestMirrorWithEmptyGitmodules(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - for i, content := range []string{"", "[submodule \"x\"]\n\tbranch = main\n"} { - if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(content), 0o600); err != nil { - t.Fatal(err) - } - runGit(t, work, "add", ".gitmodules") - runGit(t, work, "commit", "--quiet", "--allow-empty", "-m", "gitmodules "+strconv.Itoa(i)) - } - runGit(t, work, "push", "--quiet", "origin", "main") - tc := newTestCache(t, upstream.server.URL, nil) - - for _, rev := range []string{"HEAD~1", "HEAD"} { - commit := runGit(t, work, "rev-parse", rev) - got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) - if err != nil || len(got.Submodules) != 0 { - t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err) - } - } -} - -// TestSubmodulesReportsCancel checks a cancelled context is reported, not -// read as a commit without .gitmodules. -func TestSubmodulesReportsCancel(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - commit := runGit(t, work, "rev-parse", "HEAD") - ctx, cancel := context.WithCancel(context.Background()) - cancel() - tc := newTestCache(t, upstream.server.URL, nil) - - if _, err := tc.submodules(ctx, "up/repo.git", work, commit); !errors.Is(err, context.Canceled) { - t.Errorf("submodules with a cancelled context = %v, want context.Canceled", err) - } -} - -// TestMirrorSkipsBrokenGitmodules checks a .gitmodules git can't read, one -// with a syntax error or committed as a symlink, lists no submodules and -// logs a warning instead of failing the mirror. -func TestMirrorSkipsBrokenGitmodules(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - gitmodules := filepath.Join(work, ".gitmodules") - if err := os.WriteFile(gitmodules, []byte("[submodule \"x\"\n\tpath = x\n"), 0o600); err != nil { - t.Fatal(err) - } - runGit(t, work, "add", ".gitmodules") - runGit(t, work, "commit", "--quiet", "-m", "broken gitmodules") - if err := os.Remove(gitmodules); err != nil { - t.Fatal(err) - } - if err := os.Symlink("../outside/.gitmodules", gitmodules); err != nil { - t.Fatal(err) - } - runGit(t, work, "add", ".gitmodules") - runGit(t, work, "commit", "--quiet", "-m", "symlinked gitmodules") - runGit(t, work, "push", "--quiet", "origin", "main") - tc := newTestCache(t, upstream.server.URL, nil) - - for _, rev := range []string{"HEAD~1", "HEAD"} { - commit := runGit(t, work, "rev-parse", rev) - got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) - if err != nil || len(got.Submodules) != 0 { - t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err) - } - if !strings.Contains(tc.logs.String(), "commit="+commit) { - t.Errorf("no warning naming %s in the log", commit) - } - } -} - -func TestMirrorAcceptsUppercaseCommit(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - commit := runGit(t, work, "rev-parse", "HEAD") - tc := newTestCache(t, upstream.server.URL, nil) - - got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", strings.ToUpper(commit), true) - if err != nil || got.Commit != commit { - t.Errorf("mirror of the uppercase commit ID = %+v, %v, want commit %s", got, err, commit) - } -} - -func TestMirrorExplainsShortCommit(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - commit := runGit(t, work, "rev-parse", "HEAD") - tc := newTestCache(t, upstream.server.URL, nil) - - _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit[:12], true) - if !errors.Is(err, ErrNotFound) || !strings.Contains(err.Error(), "full commit ID") { - t.Errorf("mirror of a short commit ID = %v, want ErrNotFound saying it is no full commit ID", err) - } -} - -// TestMirrorWarnsOfStaleTag checks a tag served from the cache when the -// refresh fails is mirrored with a warning, since the tag may have moved. -func TestMirrorWarnsOfStaleTag(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true); err != nil { - t.Fatalf("Mirror: %v", err) - } - upstream.status.Store(http.StatusServiceUnavailable) - - if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true); err != nil { - t.Fatalf("mirror of a cached tag with the upstream down: %v", err) - } - if !strings.Contains(tc.logs.String(), "tag=v1") { - t.Errorf("no warning naming the tag in the log:\n%s", tc.logs.String()) - } -} diff --git a/internal/gitcache/pktline.go b/internal/gitcache/pktline.go deleted file mode 100644 index 5f6095b..0000000 --- a/internal/gitcache/pktline.go +++ /dev/null @@ -1,65 +0,0 @@ -package gitcache - -import ( - "bufio" - "fmt" - "io" - "strconv" - "strings" -) - -// pkt-line framing (gitprotocol-common): four hex digits giving the packet -// length including themselves, then the payload. The lengths 0000, 0001 and -// 0002 are the flush, delimiter and response-end packets. -const ( - pktFlush = "0000" - pktDelim = "0001" - pktHeaderLen = 4 - pktMaxLen = 65520 - pktSpecialLast = 2 -) - -type pktKind int - -const ( - pktData pktKind = iota - pktFlushKind - pktDelimKind - pktEndKind -) - -func pktLine(payload string) string { - return fmt.Sprintf("%04x%s", len(payload)+pktHeaderLen, payload) -} - -type pktReader struct { - r *bufio.Reader -} - -func newPktReader(r io.Reader) *pktReader { - return &pktReader{r: bufio.NewReader(r)} -} - -// next returns the next packet. A data packet's payload has its trailing -// newline removed. -func (p *pktReader) next() (pktKind, string, error) { - var header [pktHeaderLen]byte - if _, err := io.ReadFull(p.r, header[:]); err != nil { - return 0, "", err - } - n, err := strconv.ParseUint(string(header[:]), 16, 16) - if err != nil { - return 0, "", fmt.Errorf("invalid pkt-line length %q", header) - } - if n <= pktSpecialLast { - return pktKind(n + 1), "", nil - } - if n < pktHeaderLen || n > pktMaxLen { - return 0, "", fmt.Errorf("invalid pkt-line length %d", n) - } - payload := make([]byte, n-pktHeaderLen) - if _, err := io.ReadFull(p.r, payload); err != nil { - return 0, "", err - } - return pktData, strings.TrimSuffix(string(payload), "\n"), nil -} diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go deleted file mode 100644 index f467a3d..0000000 --- a/internal/gitcache/refresh.go +++ /dev/null @@ -1,402 +0,0 @@ -package gitcache - -import ( - "context" - "errors" - "fmt" - "maps" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/git-pkgs/proxy/internal/metrics" -) - -const ( - // fetchTimeout bounds one refresh, including a cold clone of a large - // repository. - fetchTimeout = time.Hour - // defaultRefCheckTimeout bounds the ref check, so an upstream that - // stalls mid-response does not hold a repository's refresh for - // fetchTimeout. - defaultRefCheckTimeout = 2 * time.Minute - // tooLargeRetryAfter is how long a repository that passed - // git.max_repo_size is left alone before it is tried again. - tooLargeRetryAfter = time.Hour -) - -// errUpstreamUnanswered marks a fetch the upstream never answered, such as -// one during an outage. -var errUpstreamUnanswered = errors.New("upstream did not answer the fetch") - -// errClosed is returned for a refresh asked for once the cache is closing. -var errClosed = errors.New("git cache is closed") - -// refreshCall is a refresh in progress, shared by every request for the -// repository that arrives while it runs. -type refreshCall struct { - done chan struct{} - err error - // checked is closed once the upstream's branches and tags have been - // compared with the cached ones, and fetching then says whether they - // differ, so a fetch follows. - checked chan struct{} - fetching bool - checkedOnce sync.Once -} - -func newRefreshCall() *refreshCall { - return &refreshCall{done: make(chan struct{}), checked: make(chan struct{})} -} - -func (call *refreshCall) markChecked(fetching bool) { - call.checkedOnce.Do(func() { - call.fetching = fetching - close(call.checked) - }) -} - -// ensureFresh brings the repository up to date with its upstream before it is -// served. A repository checked successfully within StaleMaxAge is served -// from the cache if the upstream fails or its check takes longer than -// RefreshWait; the refresh carries on in the background either way. Once the -// check finds new commits, their fetch is waited for, so a clone right after -// a push gets them. One never cloned, or not checked within StaleMaxAge, is -// served only once the refresh succeeds. -func (c *Cache) ensureFresh(ctx context.Context, rp *repo, repoURL string) error { - call := c.startRefresh(rp, repoURL) - checked := time.Unix(0, rp.lastChecked.Load()) - if !rp.exists() || time.Since(checked) > c.opts.StaleMaxAge { - select { - case <-call.done: - return call.err - case <-ctx.Done(): - return ctx.Err() - } - } - - timeout := c.opts.refreshWaitOver - if timeout == nil { - timer := time.NewTimer(c.opts.RefreshWait) - defer timer.Stop() - timeout = timer.C - } - refChecked := call.checked - for { - select { - case <-refChecked: - refChecked = nil - if call.fetching { - timeout = nil - } - case <-call.done: - if call.err != nil && servesStale(call.err) { - c.logger.Warn("git cache: refresh failed, serving cached copy", - "repository", rp.key, "checked", checked, "error", call.err) - return nil - } - return call.err - case <-timeout: - // select picks at random when the check finished just as the - // wait ran out; a check found new commits in time still counts. - select { - case <-refChecked: - refChecked = nil - if call.fetching { - timeout = nil - continue - } - default: - } - c.logger.Info("git cache: upstream check still running, serving cached copy", - "repository", rp.key, "checked", checked) - return nil - case <-ctx.Done(): - return ctx.Err() - } - } -} - -// servesStale reports whether the cached copy may be served after err: not -// when the upstream says the repository is gone or refuses access to it. -func servesStale(err error) bool { - return !errors.Is(err, ErrNotFound) && !errors.Is(err, ErrRefused) -} - -// startRefresh starts a refresh of the repository, or joins the one running. -// The refresh holds a use of the repository until it ends. -func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { - c.mu.Lock() - defer c.mu.Unlock() - if rp.refreshing != nil { - return rp.refreshing - } - call := newRefreshCall() - if c.closed { - call.err = errClosed - close(call.done) - return call - } - rp.refreshing = call - rp.users++ - c.refreshes.Add(1) - go func() { - defer c.refreshes.Done() - ctx, cancel := context.WithTimeout(c.ctx, fetchTimeout) - start := time.Now() - call.err = c.refresh(ctx, rp, repoURL, call.markChecked) - call.markChecked(false) - cancel() - metrics.RecordUpstreamFetch(ecosystem, time.Since(start)) - // A refresh cut short by Close is not an upstream failure. - if call.err != nil && c.ctx.Err() == nil { - metrics.RecordUpstreamError(ecosystem, errorType(call.err), metrics.FailureDetail{URL: repoURL, Err: call.err}) - c.logger.Warn("git cache: refresh failed", "repository", rp.key, "error", call.err) - } - c.mu.Lock() - rp.refreshing = nil - if call.err != nil && !servesStale(call.err) { - rp.refused = call.err - } - c.mu.Unlock() - c.release(rp) - close(call.done) - if call.err == nil { - c.evict() - } - }() - return call -} - -// refresh checks the upstream's branches and tags and fetches if they differ -// from the cached ones, cloning the repository if it is not cached yet. -// checked is called once a cached repository's refs have been compared. -func (c *Cache) refresh(ctx context.Context, rp *repo, repoURL string, checked func(fetching bool)) error { - if c.isTooLarge(rp.key) { - return ErrTooLarge - } - checkCtx, cancel := context.WithTimeout(ctx, c.refCheckTimeout) - remote, err := c.listUpstreamRefs(checkCtx, repoURL) - cancel() - if err != nil { - return err - } - // The upstream lets us read the repository again, even if the fetch - // below fails. - c.mu.Lock() - rp.refused = nil - c.mu.Unlock() - if rp.exists() { - err = c.update(ctx, rp, repoURL, remote, checked) - } else { - err = c.clone(ctx, rp, repoURL, remote) - } - if errors.Is(err, ErrTooLarge) { - c.markTooLarge(rp.key) - } - if err != nil { - return err - } - now := time.Now() - rp.lastChecked.Store(now.UnixNano()) - touch(rp.dir, checkedStamp, now) - return nil -} - -// clone builds the repository in a temporary directory and moves it into -// place, so no request ever reads a partial clone. -func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remoteRefs) error { - // Released after the temporary clone is removed. - res := c.newReservation() - defer res.release() - tmp, err := os.MkdirTemp(c.tmpDir(), "clone-") - if err != nil { - return err - } - defer func() { _ = os.RemoveAll(tmp) }() - - if _, err := c.run(ctx, tmp, "init", "--quiet", "--bare"); err != nil { - return err - } - if err := c.fetch(ctx, tmp, repoURL, c.opts.MaxRepoSize, res, refreshFlags, branchesAndTags...); err != nil { - return err - } - if err := c.setHead(ctx, tmp, remote, ""); err != nil { - return err - } - size := dirSize(tmp) - if c.opts.MaxRepoSize > 0 && size > c.opts.MaxRepoSize { - return ErrTooLarge - } - if err := os.MkdirAll(filepath.Dir(rp.dir), dirPermissions); err != nil { - return err - } - if err := os.Rename(tmp, rp.dir); err != nil { - return fmt.Errorf("moving clone into place: %w", err) - } - res.settle(rp, size) - c.logger.Info("git cache: cloned repository", "repository", rp.key, "size", size) - return nil -} - -// update fetches when the upstream's branches or tags differ from the cached -// ones, and follows a change of the upstream's default branch. -func (c *Cache) update(ctx context.Context, rp *repo, repoURL string, remote remoteRefs, checked func(fetching bool)) error { - unlock, err := rp.lockWrites(ctx) - if err != nil { - return err - } - defer unlock() - local, wanted, err := c.localRefs(ctx, rp.dir) - if err != nil { - return err - } - fetched := !maps.Equal(local, remote.refs) - checked(fetched) - if fetched { - limit, err := c.fetchLimit(rp) - if err != nil { - return err - } - res := c.newReservation() - err = c.fetch(ctx, rp.dir, repoURL, limit, res, refreshFlags, branchesAndTags...) - res.settle(rp, dirSize(rp.dir)) - if err != nil { - return err - } - } - if err := c.pruneWanted(ctx, rp.dir, wanted, fetched); err != nil { - c.logger.Warn("git cache: pruning wanted refs failed", "repository", rp.key, "error", err) - } - head, _ := c.run(ctx, rp.dir, "symbolic-ref", "--quiet", "HEAD") - return c.setHead(ctx, rp.dir, remote, strings.TrimSpace(head)) -} - -// branchesAndTags are the refspecs of a refresh. A bare clone has none of its -// own, and other refs, such as Gerrit's refs/changes, are left out. -var branchesAndTags = []string{"+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"} - -// refreshFlags delete the branches and tags the upstream no longer has, and -// update all refs together, so a client served while a refresh runs never -// sees only some of them moved. -var refreshFlags = []string{"--prune", "--atomic"} - -// fetchLimit returns how many more bytes the repository may take, 0 for no -// limit. -func (c *Cache) fetchLimit(rp *repo) (int64, error) { - if c.opts.MaxRepoSize <= 0 { - return 0, nil - } - limit := c.opts.MaxRepoSize - rp.size.Load() - if limit <= 0 { - return 0, ErrTooLarge - } - return limit, nil -} - -// fetch fetches refspecs from the upstream into the repository at dir, -// through the relay, adding flags to git fetch. limit caps the bytes -// fetched, 0 for no cap, and res holds space within MaxSize for them. -func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, res *reservation, flags []string, refspecs ...string) error { - select { - case c.fetches <- struct{}{}: - defer func() { <-c.fetches }() - case <-ctx.Done(): - return ctx.Err() - } - - relayURL, route, unregister := c.relay.register(ctx, repoURL, limit, res) - defer unregister() - args := append([]string{"fetch", "--quiet", "--no-write-fetch-head"}, flags...) - _, err := c.run(ctx, dir, append(append(args, relayURL), refspecs...)...) - if err != nil { - // index-pack leaves its temporary pack behind when its input ends - // early. Fetches into a repository run one at a time, so any left - // now are this fetch's. - leftovers, _ := filepath.Glob(filepath.Join(dir, "objects", "pack", "tmp_*")) - for _, path := range leftovers { - _ = os.Remove(path) - } - } - if route.exceeded.Load() { - return ErrTooLarge - } - if route.full.Load() { - return ErrCacheFull - } - if status := route.redirected.Load(); status != 0 { - return upstreamStatusError(int(status)) - } - if err != nil && !route.answered.Load() { - return fmt.Errorf("%w: %w", errUpstreamUnanswered, err) - } - return err -} - -// setHead points HEAD at the upstream's default branch, when the upstream -// names one and it is among the cached branches. -func (c *Cache) setHead(ctx context.Context, dir string, remote remoteRefs, current string) error { - if remote.head == "" || remote.head == current { - return nil - } - if _, ok := remote.refs[remote.head]; !ok { - return nil - } - _, err := c.run(ctx, dir, "symbolic-ref", "HEAD", remote.head) - return err -} - -// localRefs lists the cached branches and tags, and the names of the wanted -// refs. -func (c *Cache) localRefs(ctx context.Context, dir string) (map[string]string, []string, error) { - out, err := c.run(ctx, dir, "for-each-ref", "--format=%(objectname) %(refname)", headsPrefix, tagsPrefix, wantedPrefix) - if err != nil { - return nil, nil, err - } - refs := make(map[string]string) - var wanted []string - for line := range strings.Lines(out) { - oid, name, ok := strings.Cut(strings.TrimSuffix(line, "\n"), " ") - switch { - case !ok: - case strings.HasPrefix(name, wantedPrefix): - wanted = append(wanted, name) - default: - refs[name] = oid - } - } - return refs, wanted, nil -} - -func errorType(err error) string { - switch { - case errors.Is(err, ErrNotFound): - return "not_found" - case errors.Is(err, ErrRefused): - return "refused" - case errors.Is(err, ErrTooLarge): - return "too_large" - case errors.Is(err, ErrCacheFull): - return "cache_full" - } - return "fetch_failed" -} - -func (c *Cache) isTooLarge(key string) bool { - c.mu.Lock() - defer c.mu.Unlock() - since, ok := c.tooLarge[key] - if ok && time.Since(since) >= tooLargeRetryAfter { - delete(c.tooLarge, key) - return false - } - return ok -} - -func (c *Cache) markTooLarge(key string) { - c.mu.Lock() - defer c.mu.Unlock() - c.tooLarge[key] = time.Now() -} diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go deleted file mode 100644 index bd65b25..0000000 --- a/internal/gitcache/relay.go +++ /dev/null @@ -1,206 +0,0 @@ -package gitcache - -import ( - "context" - "crypto/rand" - "encoding/hex" - "errors" - "io" - "log/slog" - "net" - "net/http" - "strings" - "sync" - "sync/atomic" - "time" -) - -const ( - relayIDBytes = 16 - relayReadHeaderTimeout = 30 * time.Second -) - -// relay is a loopback HTTP server that every git fetch the cache runs goes -// through. git never reaches the network itself: each fetch gets a route to -// one upstream repository, and the relay forwards its requests with the -// proxy's upstream client, so address checks, upstream.auth, the access log -// and the no-redirect rule apply as they do to the proxy's own requests, and -// git never sees a credential. -type relay struct { - client *http.Client - logger *slog.Logger - listener net.Listener - server *http.Server - base string - - mu sync.Mutex - routes map[string]*relayRoute -} - -type relayRoute struct { - repoURL string - // ctx ends when the fetch does, which stops its relayed requests, so a - // git helper left behind by a killed git does not keep downloading. - ctx context.Context - // limit caps the response bytes the route relays, 0 for no cap, and - // budget, when set, holds space within MaxSize for them. exceeded and - // full record which of them stopped the route. - limit int64 - budget *reservation - relayed atomic.Int64 - exceeded atomic.Bool - full atomic.Bool - // answered records that the upstream answered a git-upload-pack - // request, so a failed fetch was not an outage. - answered atomic.Bool - // redirected records the status of a redirect the upstream answered - // with, 0 for none. - redirected atomic.Int32 -} - -func newRelay(client *http.Client, logger *slog.Logger) (*relay, error) { - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - if listener, err = net.Listen("tcp", "[::1]:0"); err != nil { - return nil, err - } - } - rl := &relay{ - client: client, - logger: logger, - listener: listener, - base: "http://" + listener.Addr().String(), - routes: make(map[string]*relayRoute), - } - rl.server = &http.Server{Handler: rl, ReadHeaderTimeout: relayReadHeaderTimeout} - go func() { - if err := rl.server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { - logger.Error("git relay stopped", "error", err) - } - }() - return rl, nil -} - -func (rl *relay) close() error { - return rl.server.Close() -} - -// register adds a route to repoURL for a fetch running with ctx and returns -// the URL git fetches from and a function that removes the route and stops -// its requests. -func (rl *relay) register(ctx context.Context, repoURL string, limit int64, budget *reservation) (string, *relayRoute, func()) { - var b [relayIDBytes]byte - _, _ = rand.Read(b[:]) - id := hex.EncodeToString(b[:]) - ctx, cancel := context.WithCancel(ctx) - route := &relayRoute{repoURL: repoURL, ctx: ctx, limit: limit, budget: budget} - - rl.mu.Lock() - rl.routes[id] = route - rl.mu.Unlock() - - return rl.base + "/" + id, route, func() { - rl.mu.Lock() - delete(rl.routes, id) - rl.mu.Unlock() - cancel() - } -} - -func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { - id, suffix, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), "/") - rl.mu.Lock() - route := rl.routes[id] - rl.mu.Unlock() - if route == nil { - http.NotFound(w, r) - return - } - - var target string - switch { - case suffix == "info/refs" && r.Method == http.MethodGet && r.URL.RawQuery == "service=git-upload-pack": - target = route.repoURL + "/info/refs?service=git-upload-pack" - case suffix == "git-upload-pack" && r.Method == http.MethodPost: - target = route.repoURL + "/git-upload-pack" - default: - http.NotFound(w, r) - return - } - - ctx, cancel := context.WithCancel(r.Context()) - defer cancel() - defer context.AfterFunc(route.ctx, cancel)() - out, err := http.NewRequestWithContext(ctx, r.Method, target, r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - out.ContentLength = r.ContentLength - for _, name := range []string{"Git-Protocol", "Content-Type", "Content-Encoding", "Accept", "User-Agent"} { - if value := r.Header.Get(name); value != "" { - out.Header.Set(name, value) - } - } - - resp, err := rl.client.Do(out) - if err != nil { - if ctx.Err() == nil { - rl.logger.Warn("git relay: upstream request failed", "upstream", route.repoURL, "error", err) - } - http.Error(w, "upstream request failed", http.StatusBadGateway) - return - } - defer func() { _ = resp.Body.Close() }() - // A redirect passed on would send git to the Location itself. - if resp.StatusCode >= http.StatusMultipleChoices && resp.StatusCode < http.StatusBadRequest { - route.redirected.Store(int32(resp.StatusCode)) - http.Error(w, "upstream redirect not followed", http.StatusBadGateway) - return - } - - if r.Method == http.MethodPost && resp.StatusCode == http.StatusOK { - route.answered.Store(true) - } - if ct := resp.Header.Get("Content-Type"); ct != "" { - w.Header().Set("Content-Type", ct) - } - w.WriteHeader(resp.StatusCode) - if _, err := io.Copy(w, &routeReader{ctx: ctx, r: resp.Body, route: route}); err != nil { - if r.Context().Err() != nil { - // git went away; there is no one to tell. - return - } - if ctx.Err() == nil && !route.exceeded.Load() && !route.full.Load() { - rl.logger.Warn("git relay: copying upstream response", "upstream", route.repoURL, "error", err) - } - // Cut the connection so git fails rather than taking a response - // that ended early for one that is complete. - panic(http.ErrAbortHandler) - } -} - -// routeReader counts relayed bytes against the route's limit and budget. -type routeReader struct { - ctx context.Context - r io.Reader - route *relayRoute -} - -func (rr *routeReader) Read(p []byte) (int, error) { - n, err := rr.r.Read(p) - total := rr.route.relayed.Add(int64(n)) - if rr.route.limit > 0 && total > rr.route.limit { - rr.route.exceeded.Store(true) - return 0, ErrTooLarge - } - if rr.route.budget != nil { - if err := rr.route.budget.cover(rr.ctx, total); err != nil { - if errors.Is(err, ErrCacheFull) { - rr.route.full.Store(true) - } - return 0, err - } - } - return n, err -} diff --git a/internal/gitcache/relay_test.go b/internal/gitcache/relay_test.go deleted file mode 100644 index 309e727..0000000 --- a/internal/gitcache/relay_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package gitcache - -import ( - "io" - "log/slog" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" -) - -func newTestRelay(t *testing.T) *relay { - t.Helper() - rl, err := newRelay(&http.Client{}, slog.New(slog.DiscardHandler)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = rl.close() }) - return rl -} - -func TestRelayOnlyForwardsFetchRequests(t *testing.T) { - var forwarded atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - forwarded.Add(1) - })) - defer upstream.Close() - rl := newTestRelay(t) - base, _, unregister := rl.register(t.Context(), upstream.URL+"/repo", 0, nil) - - refused := []struct{ method, url string }{ - {http.MethodGet, rl.base + "/unknown/info/refs?service=git-upload-pack"}, - {http.MethodGet, base + "/info/refs?service=git-receive-pack"}, - {http.MethodPost, base + "/git-receive-pack"}, - {http.MethodGet, base + "/HEAD"}, - {http.MethodGet, base + "/git-upload-pack"}, - } - for _, r := range refused { - req, _ := http.NewRequest(r.method, r.url, nil) - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusNotFound { - t.Errorf("%s %s: status %d, want 404", r.method, r.url, resp.StatusCode) - } - } - if got := forwarded.Load(); got != 0 { - t.Errorf("forwarded %d refused requests", got) - } - - unregister() - resp, err := http.Get(base + "/info/refs?service=git-upload-pack") - if err != nil { - t.Fatal(err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusNotFound || forwarded.Load() != 0 { - t.Errorf("unregistered route: status %d, forwarded %d", resp.StatusCode, forwarded.Load()) - } -} - -// TestRelayStopsAtLimit checks a fetch over git.max_repo_size is cut off -// rather than downloaded in full and thrown away. -func TestRelayStopsAtLimit(t *testing.T) { - const size, limit = 4 << 20, 64 << 10 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.Copy(io.Discard, r.Body) - _, _ = io.Copy(w, io.LimitReader(zeros{}, size)) - })) - defer upstream.Close() - var logs strings.Builder - var logMu sync.Mutex - rl, err := newRelay(&http.Client{}, slog.New(slog.NewTextHandler(lockedWriter{&logMu, &logs}, nil))) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = rl.close() }) - base, route, unregister := rl.register(t.Context(), upstream.URL+"/repo", limit, nil) - defer unregister() - - resp, err := http.Post(base+"/git-upload-pack", requestType, strings.NewReader(pktFlush)) - if err != nil { - t.Fatal(err) - } - n, err := io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - if err == nil { - t.Errorf("read %d bytes without an error", n) - } - if !route.exceeded.Load() || route.relayed.Load() > size/2 { - logMu.Lock() - defer logMu.Unlock() - t.Errorf("exceeded %v after relaying %d of %d bytes; relay log:\n%s", route.exceeded.Load(), route.relayed.Load(), size, logs.String()) - } -} - -type lockedWriter struct { - mu *sync.Mutex - w io.Writer -} - -func (l lockedWriter) Write(p []byte) (int, error) { - l.mu.Lock() - defer l.mu.Unlock() - return l.w.Write(p) -} - -type zeros struct{} - -func (zeros) Read(p []byte) (int, error) { - clear(p) - return len(p), nil -} diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go deleted file mode 100644 index 5d80938..0000000 --- a/internal/gitcache/serve.go +++ /dev/null @@ -1,351 +0,0 @@ -package gitcache - -import ( - "bytes" - "compress/gzip" - "context" - "errors" - "io" - "math" - "net/http" - "os" - "path/filepath" - "regexp" - "slices" - "strings" - "sync/atomic" - "time" -) - -// gitProtocolPattern is what a Git-Protocol header may hold, such as -// "version=2". It is passed to upload-pack in its environment. -var gitProtocolPattern = regexp.MustCompile(`^[A-Za-z0-9=:._-]*$`) - -var errInvalidKey = errors.New("invalid repository key") - -// Lease is a use of a cached repository. The repository is not evicted -// until the lease is released. -type Lease struct { - cache *Cache - repo *repo - repoURL string - hit bool - fetchWants bool -} - -// OpenOptions says what Open does besides returning the repository. -type OpenOptions struct { - // Refresh brings a cached repository up to date with the upstream first, - // or leaves it as it is when the upstream fails or is slow and the copy - // is recent enough (see Options). - Refresh bool - // FetchWants lets ServeUploadPack fetch commits a client asks for by ID - // that the cache lacks. Leave it off where a fetch by ID could reach - // commits the client may not read through this repository, such as - // GitHub forks with credentials configured. - FetchWants bool - // NoCheck leaves a cached repository as it is, for a caller that - // refreshes it itself. Otherwise a cached repository is refused when the - // last refresh found it gone or refused upstream, and refreshed first - // when it was not checked within StaleMaxAge. - NoCheck bool -} - -// Open returns a lease on the repository cached under key, a relative path -// such as github-1a2b3c4d/acme/widgets.git, whose upstream is repoURL. A -// repository not cached yet is cloned first. -func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) (*Lease, error) { - if !strings.HasSuffix(key, ".git") || !filepath.IsLocal(filepath.FromSlash(key)) { - return nil, errInvalidKey - } - rp := c.acquire(key) - hit := rp.exists() - var err error - switch { - case opts.Refresh || !hit: - err = c.ensureFresh(ctx, rp, repoURL) - if err == nil && hit && !opts.NoCheck { - // A refresh still running or failed serves the cached copy, - // but not one the upstream refused since and no check has - // cleared. - err = c.refusal(rp) - } - case !opts.NoCheck: - err = c.checkServable(ctx, rp, repoURL) - } - if err != nil { - c.release(rp) - return nil, err - } - return &Lease{cache: c, repo: rp, repoURL: repoURL, hit: hit, fetchWants: opts.FetchWants}, nil -} - -// uploadPackGrace is how long past StaleMaxAge a copy is still served -// without a check to the upload-pack requests that follow a client's -// info/refs, so a clone checks the upstream once. -const uploadPackGrace = time.Minute - -// checkServable refreshes a cached repository first when the last refresh -// found it gone or refused upstream, or it was not checked recently, and -// refuses it while that refusal stands. -func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) error { - // The sum saturates rather than overflow for a huge StaleMaxAge. - maxAge := c.opts.StaleMaxAge + uploadPackGrace - if maxAge < c.opts.StaleMaxAge { - maxAge = math.MaxInt64 - } - if c.refusal(rp) == nil && time.Since(time.Unix(0, rp.lastChecked.Load())) <= maxAge { - return nil - } - if err := c.ensureFresh(ctx, rp, repoURL); err != nil { - return err - } - return c.refusal(rp) -} - -// refusal returns the error of the last refresh that found the repository -// gone or refused upstream, until a ref check succeeds. -func (c *Cache) refusal(rp *repo) error { - c.mu.Lock() - defer c.mu.Unlock() - return rp.refused -} - -// Hit reports whether the repository was already cached when it was opened. -func (l *Lease) Hit() bool { - return l.hit -} - -// Release ends the lease. -func (l *Lease) Release() { - l.cache.release(l.repo) -} - -// ServeInfoRefs answers GET info/refs?service=git-upload-pack. -func (l *Lease) ServeInfoRefs(w http.ResponseWriter, r *http.Request) { - c := l.cache - if !takeSlot(r.Context(), c.refListings) { - return - } - defer func() { <-c.refListings }() - protocol := gitProtocol(r) - setNoCache(w) - w.Header().Set("Content-Type", advertisementType) - // Protocol v2 starts with its version line; earlier versions with the - // service name. - if !isProtocolV2(protocol) { - _, _ = io.WriteString(w, pktLine("# service=git-upload-pack\n")+pktFlush) - } - c.uploadPack(w, r, l.repo, protocol, nil, "--advertise-refs") -} - -// ServeUploadPack answers POST git-upload-pack with req, the request's body -// as ReadRequest read it. With OpenOptions.FetchWants, commits the request -// wants that the cache lacks are fetched from the upstream first. -func (l *Lease) ServeUploadPack(w http.ResponseWriter, r *http.Request, req *Request) { - c := l.cache - body := req.reader() - if req.gzip { - gz, err := gzip.NewReader(body) - if err != nil { - http.Error(w, "invalid gzip body", http.StatusBadRequest) - return - } - defer func() { _ = gz.Close() }() - // A gzip stream of any length can decode to almost nothing. - decoded := &bodyLimit{r: gz, remaining: c.maxRequestBody} - defer func() { - if decoded.exceeded.Load() { - c.logger.Warn("git cache: request body passed the limit", - "repository", l.repo.key, "limit", c.maxRequestBody) - } - }() - body = decoded - } - if l.fetchWants { - var wants []string - wants, body = readWants(body) - if err := c.fetchMissingWants(r.Context(), l.repo, l.repoURL, wants); err != nil && r.Context().Err() == nil { - // upload-pack reports the missing objects to the client. - c.logger.Warn("git cache: fetching wanted objects failed", "repository", l.repo.key, "error", err) - } - } - var command string - command, body = requestCommand(body) - // ls-refs builds no pack, so it waits for a ref listing slot rather than - // behind clones. - slots := c.uploadPacks - if command == "ls-refs" { - slots = c.refListings - } - if !takeSlot(r.Context(), slots) { - return - } - defer func() { <-slots }() - setNoCache(w) - w.Header().Set("Content-Type", resultType) - c.uploadPack(w, r, l.repo, gitProtocol(r), body) -} - -// takeSlot takes a slot from slots, or reports false once ctx is done. -func takeSlot(ctx context.Context, slots chan struct{}) bool { - select { - case slots <- struct{}{}: - return true - case <-ctx.Done(): - return false - } -} - -// requestCommand returns the protocol v2 command an upload-pack request -// starts with, such as "fetch", or "" for an earlier protocol, with a reader -// that replays the whole body. -func requestCommand(body io.Reader) (string, io.Reader) { - var consumed bytes.Buffer - _, line, _ := newPktReader(io.TeeReader(body, &consumed)).next() - command, ok := strings.CutPrefix(line, "command=") - if !ok { - command = "" - } - return command, io.MultiReader(&consumed, body) -} - -// servingConfig gives each pack-objects that upload-pack runs one -// delta-search thread with a bounded window, where git defaults to a thread -// per CPU and no bound. -var servingConfig = []string{"-c", "pack.threads=1", "-c", "pack.windowMemory=64m"} - -// uploadPack runs git upload-pack on the cached repository in stateless mode, -// the way git http-backend does. -func (c *Cache) uploadPack(w http.ResponseWriter, r *http.Request, rp *repo, protocol string, stdin io.Reader, args ...string) { - var env []string - if protocol != "" { - env = append(env, "GIT_PROTOCOL="+protocol) - } - args = slices.Concat(servingConfig, []string{"upload-pack", "--stateless-rpc"}, args, []string{rp.dir}) - cmd := c.command(r.Context(), env, args...) - cmd.Stdin = stdin - cmd.Stdout = w - stderr := &limitedBuffer{max: maxStderr} - cmd.Stderr = stderr - if err := cmd.Run(); err != nil && r.Context().Err() == nil { - c.logger.Warn("git cache: upload-pack failed", - "repository", rp.key, "error", err, "stderr", strings.TrimSpace(stderr.String())) - } -} - -// defaultMaxRequestBody bounds an upload-pack request far above any real -// negotiation, so a small gzip body cannot feed upload-pack for as long as -// the request may run. -const defaultMaxRequestBody = 100 << 20 - -// maxMemoryRequest is the most of a request body kept in memory; the rest of -// a larger one goes to a temporary file. -const maxMemoryRequest = 1 << 20 - -// ErrRequestTooLarge is returned by ReadRequest for a body past the limit. -var ErrRequestTooLarge = errors.New("request body too large") - -// Request is the body of a POST git-upload-pack request, read in full -// before anything is started for it. Stateless requests are complete once -// sent, so a client that sends one slowly, or not at all, holds no -// upload-pack process and is bound by the server's read timeout. -type Request struct { - gzip bool - data []byte - file *os.File -} - -// ReadRequest reads the body of a POST git-upload-pack request, still -// encoded. A body past the limit fails with ErrRequestTooLarge. Close the -// request once it is served. -func (c *Cache) ReadRequest(r *http.Request) (*Request, error) { - req := &Request{gzip: r.Header.Get("Content-Encoding") == "gzip"} - body := &bodyLimit{r: r.Body, remaining: c.maxRequestBody} - var buf bytes.Buffer - if _, err := buf.ReadFrom(io.LimitReader(body, maxMemoryRequest)); err != nil { - return nil, err - } - if buf.Len() < maxMemoryRequest { - req.data = buf.Bytes() - return req, nil - } - file, err := os.CreateTemp(c.tmpDir(), "git-upload-pack-*") - if err != nil { - return nil, err - } - req.file = file - if _, err := buf.WriteTo(file); err != nil { - req.Close() - return nil, err - } - if _, err := io.Copy(file, body); err != nil { - req.Close() - return nil, err - } - return req, nil -} - -// Close removes any temporary file holding the request. -func (req *Request) Close() { - if req.file != nil { - _ = req.file.Close() - _ = os.Remove(req.file.Name()) - } -} - -func (req *Request) reader() io.Reader { - if req.file != nil { - return io.NewSectionReader(req.file, 0, 1<<63-1) - } - return bytes.NewReader(req.data) -} - -// bodyLimit fails reads past remaining bytes, where io.LimitReader would end -// the body quietly and leave upload-pack a request cut short. -type bodyLimit struct { - r io.Reader - remaining int64 - exceeded atomic.Bool -} - -func (b *bodyLimit) Read(p []byte) (int, error) { - if b.exceeded.Load() { - return 0, ErrRequestTooLarge - } - // One byte past the limit tells a longer body from one that ends at it. - if int64(len(p)) > b.remaining+1 { - p = p[:b.remaining+1] - } - n, err := b.r.Read(p) - if int64(n) <= b.remaining { - b.remaining -= int64(n) - return n, err - } - n = int(b.remaining) - b.remaining = 0 - b.exceeded.Store(true) - return n, ErrRequestTooLarge -} - -func gitProtocol(r *http.Request) string { - if protocol := r.Header.Get("Git-Protocol"); gitProtocolPattern.MatchString(protocol) { - return protocol - } - return "" -} - -func isProtocolV2(protocol string) bool { - for field := range strings.SplitSeq(protocol, ":") { - if field == "version=2" { - return true - } - } - return false -} - -func setNoCache(w http.ResponseWriter) { - w.Header().Set("Expires", "Fri, 01 Jan 1980 00:00:00 GMT") - w.Header().Set("Pragma", "no-cache") - w.Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate") -} diff --git a/internal/gitcache/serve_test.go b/internal/gitcache/serve_test.go deleted file mode 100644 index ca53e40..0000000 --- a/internal/gitcache/serve_test.go +++ /dev/null @@ -1,431 +0,0 @@ -package gitcache - -import ( - "bytes" - "compress/gzip" - "errors" - "fmt" - "io" - "net/http" - "os" - "os/exec" - "path/filepath" - "slices" - "strings" - "sync" - "testing" - "time" -) - -// TestRequestBodies sends the bodies git sends for large requests: gzipped, -// or chunked once one outgrows http.postBuffer. -func TestRequestBodies(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - - lsRefs := pktLine("command=ls-refs\n") + pktDelim + pktLine("ref-prefix refs/heads/\n") + pktFlush - var gzipped bytes.Buffer - gz := gzip.NewWriter(&gzipped) - _, _ = io.WriteString(gz, lsRefs) - _ = gz.Close() - for _, tt := range []struct { - name string - body io.Reader - encoding string - }{ - // A reader of unknown length is sent chunked. - {"chunked", io.MultiReader(strings.NewReader(lsRefs)), ""}, - {"gzip", &gzipped, "gzip"}, - } { - req, _ := http.NewRequest(http.MethodPost, tc.url("repo")+"/git-upload-pack", tt.body) - req.Header.Set("Content-Type", requestType) - req.Header.Set("Git-Protocol", "version=2") - if tt.encoding != "" { - req.Header.Set("Content-Encoding", tt.encoding) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } - out, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if resp.StatusCode != http.StatusOK || !strings.Contains(string(out), "refs/heads/main") { - t.Errorf("%s: status %d, body %q", tt.name, resp.StatusCode, out) - } - } -} - -// TestRequestBodyLimit refuses a request past the limit, before or after -// gzip, including one that is small compressed. -func TestRequestBodyLimit(t *testing.T) { - const limit = 4096 - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.maxRequestBody = limit }) - tc.clone(t, "repo") - - lsRefs := func(prefixes int) string { - return pktLine("command=ls-refs\n") + pktDelim + - strings.Repeat(pktLine("ref-prefix refs/heads/\n"), prefixes) + pktFlush - } - gzipped := func(body string) *bytes.Buffer { - var buf bytes.Buffer - gz := gzip.NewWriter(&buf) - _, _ = io.WriteString(gz, body) - _ = gz.Close() - return &buf - } - small, large := lsRefs(10), lsRefs(200) - if len(small) > limit || len(large) <= limit { - t.Fatalf("bodies of %d and %d bytes do not straddle the limit", len(small), len(large)) - } - compressed := gzipped(large) - if compressed.Len() > limit { - t.Fatalf("compressed body is %d bytes, over the limit", compressed.Len()) - } - // Empty gzip members before a small body: long on the wire, short decoded. - var padded bytes.Buffer - for empty := gzipped(""); padded.Len() <= limit; { - padded.Write(empty.Bytes()) - } - padded.Write(gzipped(small).Bytes()) - for _, tt := range []struct { - name string - body io.Reader - encoding string - served bool - }{ - {"small", strings.NewReader(small), "", true}, - {"small gzip", gzipped(small), "gzip", true}, - {"large", strings.NewReader(large), "", false}, - {"large gzip", compressed, "gzip", false}, - {"padded gzip", &padded, "gzip", false}, - } { - req, _ := http.NewRequest(http.MethodPost, tc.url("repo")+"/git-upload-pack", tt.body) - req.Header.Set("Content-Type", requestType) - req.Header.Set("Git-Protocol", "version=2") - if tt.encoding != "" { - req.Header.Set("Content-Encoding", tt.encoding) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } - out, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if served := strings.Contains(string(out), "refs/heads/main"); served != tt.served { - t.Errorf("%s: served %t, want %t (status %d, body %q)", tt.name, served, tt.served, resp.StatusCode, out) - } - } -} - -// TestUploadPackSlots holds the only upload-pack slot: a clone waits for it, -// while listing refs, which builds no pack, does not. -func TestUploadPackSlots(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.maxUploadPacks = 1 }) - tc.clone(t, "repo") - - start := func(args ...string) <-chan error { - done := make(chan error, 1) - go func() { - out, err := tryGit(t, "", args...) - if err != nil { - err = fmt.Errorf("%w\n%s", err, out) - } - done <- err - }() - return done - } - - tc.uploadPacks <- struct{}{} - held := true - release := func() { - if held { - held = false - <-tc.uploadPacks - } - } - // A failure while the slot is held must free it for the cache to close. - defer release() - select { - case err := <-start("-c", "protocol.version=2", "ls-remote", tc.url("repo")): - if err != nil { - t.Fatalf("ls-remote: %v", err) - } - case <-time.After(10 * time.Second): - t.Fatal("ls-remote waited for the slot") - } - clone := start("clone", "--quiet", tc.url("repo"), filepath.Join(t.TempDir(), "clone")) - select { - case err := <-clone: - t.Fatalf("clone finished while the slot was held: %v", err) - case <-time.After(time.Second): - } - release() - if err := <-clone; err != nil { - t.Errorf("clone: %v", err) - } -} - -// TestRefListingSlots holds the only slot for listing refs: info/refs and -// ls-refs wait for it. -func TestRefListingSlots(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.maxRefListings = 1 }) - tc.clone(t, "repo") - - post := func(body string) <-chan error { - done := make(chan error, 1) - go func() { - req, _ := http.NewRequest(http.MethodPost, tc.url("repo")+"/git-upload-pack", strings.NewReader(body)) - req.Header.Set("Content-Type", requestType) - req.Header.Set("Git-Protocol", "version=2") - resp, err := http.DefaultClient.Do(req) - if err != nil { - done <- err - return - } - out, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if !strings.Contains(string(out), "refs/heads/main") { - err = fmt.Errorf("status %d, body %q", resp.StatusCode, out) - } - done <- err - }() - return done - } - get := func() <-chan error { - done := make(chan error, 1) - go func() { - resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") - if err != nil { - done <- err - return - } - out, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if !strings.Contains(string(out), "refs/heads/main") { - err = fmt.Errorf("status %d, body %q", resp.StatusCode, out) - } - done <- err - }() - return done - } - - tc.refListings <- struct{}{} - held := true - release := func() { - if held { - held = false - <-tc.refListings - } - } - defer release() - lsRefs := post(pktLine("command=ls-refs\n") + pktDelim + pktLine("ref-prefix refs/heads/\n") + pktFlush) - infoRefs := get() - select { - case err := <-lsRefs: - t.Fatalf("ls-refs finished while the slot was held: %v", err) - case err := <-infoRefs: - t.Fatalf("info/refs finished while the slot was held: %v", err) - case <-time.After(time.Second): - } - release() - for name, done := range map[string]<-chan error{"ls-refs": lsRefs, "info/refs": infoRefs} { - if err := <-done; err != nil { - t.Errorf("%s: %v", name, err) - } - } -} - -// TestReadRequest reads bodies kept in memory and, past maxMemoryRequest, in -// a temporary file removed on Close. -func TestReadRequest(t *testing.T) { - c := &Cache{dir: t.TempDir(), maxRequestBody: 4 << 20} - if err := os.Mkdir(c.tmpDir(), dirPermissions); err != nil { - t.Fatal(err) - } - for _, size := range []int{0, 100, maxMemoryRequest, 3 << 20} { - body := strings.Repeat("x", size) - r, _ := http.NewRequest(http.MethodPost, "/", strings.NewReader(body)) - req, err := c.ReadRequest(r) - if err != nil { - t.Fatalf("%d bytes: %v", size, err) - } - if got, _ := io.ReadAll(req.reader()); string(got) != body { - t.Errorf("%d bytes: read back %d", size, len(got)) - } - req.Close() - if req.file != nil { - if _, err := os.Stat(req.file.Name()); !errors.Is(err, os.ErrNotExist) { - t.Errorf("%d bytes: temporary file left: %v", size, err) - } - } - } - r, _ := http.NewRequest(http.MethodPost, "/", strings.NewReader(strings.Repeat("x", 5<<20))) - if _, err := c.ReadRequest(r); !errors.Is(err, ErrRequestTooLarge) { - t.Errorf("body past the limit: %v, want ErrRequestTooLarge", err) - } -} - -func TestRequestCommand(t *testing.T) { - for _, tt := range []struct{ body, want string }{ - {pktLine("command=ls-refs\n") + pktDelim + pktFlush, "ls-refs"}, - {pktLine("command=fetch\n") + pktDelim + pktLine("done\n") + pktFlush, "fetch"}, - {pktLine("want "+strings.Repeat("a", 40)+"\n") + pktFlush, ""}, - {"", ""}, - } { - command, body := requestCommand(strings.NewReader(tt.body)) - replayed, _ := io.ReadAll(body) - if command != tt.want || string(replayed) != tt.body { - t.Errorf("requestCommand(%q) = %q, %q; want %q and the whole body", tt.body, command, replayed, tt.want) - } - } -} - -// TestCloneWithChunkedRequests clones a repository with enough refs that -// git's requests, to the cache and from the cache to the upstream, outgrow -// http.postBuffer and are sent chunked. -func TestCloneWithChunkedRequests(t *testing.T) { - const tags = 2000 - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - var stream strings.Builder - for i := 1; i <= tags; i++ { - fmt.Fprintf(&stream, "commit refs/heads/main\nmark :%d\ncommitter test %d +0000\ndata 1\n\n", i, i) - if i == 1 { - stream.WriteString("from refs/heads/main^0\n") - } - fmt.Fprintf(&stream, "reset refs/tags/t%d\nfrom :%d\n\n", i, i) - } - importer := exec.Command("git", "fast-import", "--quiet") - importer.Dir, importer.Env, importer.Stdin = work, gitEnv(t.TempDir()), strings.NewReader(stream.String()) - if out, err := importer.CombinedOutput(); err != nil { - t.Fatalf("git fast-import: %v\n%s", err, out) - } - runGit(t, work, "push", "--quiet", "origin", "main", "--tags") - tc := newTestCache(t, upstream.server.URL, nil) - - // One want per tag is about 100KB, past a postBuffer just above git's - // minimum. - clone := tc.clone(t, "repo", "-c", "http.postBuffer=66000") - if got := strings.Count(runGit(t, clone, "tag", "--list"), "\n") + 1; got != tags+1 { - t.Errorf("cloned %d tags, want %d", got, tags+1) - } -} - -func TestParseV0Advertisement(t *testing.T) { - oid := strings.Repeat("a", 40) - peeled := strings.Repeat("b", 40) - adv := pktLine(oid+" HEAD\x00multi_ack symref=HEAD:refs/heads/trunk agent=git/2\n") + - pktLine(oid+" refs/heads/trunk\n") + - pktLine(oid+" refs/tags/v1\n") + - pktLine(peeled+" refs/tags/v1^{}\n") + - pktLine(oid+" refs/changes/01/1/1\n") + - pktLine(oid+" refs/pull/1/head\n") + - pktFlush - pkts := newPktReader(strings.NewReader(adv)) - kind, line, err := pkts.next() - if err != nil { - t.Fatal(err) - } - remote, err := parseV0Advertisement(pkts, kind, line) - if err != nil { - t.Fatal(err) - } - if remote.head != "refs/heads/trunk" { - t.Errorf("head = %q", remote.head) - } - want := map[string]string{"refs/heads/trunk": oid, "refs/tags/v1": oid} - if len(remote.refs) != len(want) || remote.refs["refs/heads/trunk"] != oid || remote.refs["refs/tags/v1"] != oid { - t.Errorf("refs = %v, want %v", remote.refs, want) - } -} - -// TestCommitOnNoBranchIsFetchedByID does what actions/checkout does: fetch -// one commit by object ID, here a pull request head that is on no branch. -func TestCommitOnNoBranchIsFetchedByID(t *testing.T) { - for _, version := range []string{"0", "2"} { - t.Run("protocol v"+version, func(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - runGit(t, work, "checkout", "--quiet", "-b", "pr") - sha := upstream.commit(work, "pull request") - runGit(t, work, "push", "--quiet", "origin", "pr:refs/pull/1/head") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - sessions := upstream.gitSessions.Load() - - var wg sync.WaitGroup - for range 4 { - wg.Go(func() { - dir := t.TempDir() - runGit(t, dir, "init", "--quiet") - if out, err := tryGit(t, dir, "-c", "protocol.version="+version, "fetch", "--quiet", "--depth=1", tc.url("repo"), sha); err != nil { - t.Errorf("fetch by ID: %v\n%s", err, out) - return - } - if got := runGit(t, dir, "rev-parse", "FETCH_HEAD"); got != sha { - t.Errorf("FETCH_HEAD = %s, want %s", got, sha) - } - }) - } - wg.Wait() - if got := upstream.gitSessions.Load() - sessions; got != 1 { - t.Errorf("upstream saw %d fetches for the commit, want 1", got) - } - if refs := runGit(t, "", "ls-remote", tc.url("repo")); strings.Contains(refs, wantedPrefix) { - t.Errorf("advertised %s refs:\n%s", wantedPrefix, refs) - } - - // A refresh that prunes branches keeps the commit. - runGit(t, work, "push", "--quiet", "origin", "main:refs/heads/new") - tc.clone(t, "repo") - if refs := tc.cachedRefs(t); !strings.Contains(refs, wantedPrefix) || !strings.Contains(refs, "/"+sha) { - t.Errorf("refresh dropped the fetched commit:\n%s", refs) - } - }) - } -} - -func TestReadWants(t *testing.T) { - a, b := strings.Repeat("a", 40), strings.Repeat("b", 40) - tests := map[string]struct { - body string - want []string - }{ - "v0": { - pktLine("want "+a+" multi_ack side-band-64k\n") + pktLine("want "+b+"\n") + pktLine("deepen 1\n") + pktFlush + pktLine("have "+a+"\n") + pktFlush, - []string{a, b}, - }, - "v2 fetch": { - pktLine("command=fetch\n") + pktLine("agent=git/2\n") + pktDelim + pktLine("thin-pack\n") + pktLine("want "+a+"\n") + pktLine("have "+b+"\n") + pktLine("want "+b+"\n") + pktLine("done\n") + pktFlush, - []string{a, b}, - }, - "v2 ls-refs": { - pktLine("command=ls-refs\n") + pktDelim + pktLine("want "+a+"\n") + pktFlush, - nil, - }, - "not an object ID": { - pktLine("want HEAD\n") + pktFlush, - nil, - }, - } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - wants, replay := readWants(strings.NewReader(tt.body)) - if !slices.Equal(wants, tt.want) { - t.Errorf("wants = %v, want %v", wants, tt.want) - } - if got, _ := io.ReadAll(replay); string(got) != tt.body { - t.Errorf("replayed %q, want the whole body", got) - } - }) - } -} diff --git a/internal/gitcache/upstream.go b/internal/gitcache/upstream.go deleted file mode 100644 index 46a2667..0000000 --- a/internal/gitcache/upstream.go +++ /dev/null @@ -1,214 +0,0 @@ -package gitcache - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "strings" -) - -const ( - advertisementType = "application/x-git-upload-pack-advertisement" - requestType = "application/x-git-upload-pack-request" - resultType = "application/x-git-upload-pack-result" - // Some servers only speak the smart protocol to clients that look like git. - gitUserAgent = "git/2 (git-pkgs-proxy)" - // maxRefsResponse bounds a ref listing; branches and tags of even large - // repositories take a few megabytes. - maxRefsResponse = 64 << 20 - - headsPrefix = "refs/heads/" - tagsPrefix = "refs/tags/" -) - -var ( - // ErrNotFound is returned when the upstream has no such repository. - ErrNotFound = errors.New("repository not found upstream") - // ErrRefused covers authentication failures and redirects. The cached - // copy is not served after one, since the upstream may have withdrawn - // access to it. - ErrRefused = errors.New("upstream refused the request") - // ErrTooLarge is returned for a repository larger than git.max_repo_size, - // or than git.max_size where that is smaller. - ErrTooLarge = errors.New("repository is larger than git.max_repo_size or git.max_size") - // ErrCacheFull is returned for a fetch cut off because repositories in - // use take what git.max_size leaves. - ErrCacheFull = errors.New("git cache is full") -) - -// remoteRefs is what a repository advertises: its branch and tag tips, and -// the branch HEAD points at. -type remoteRefs struct { - refs map[string]string - head string -} - -// listUpstreamRefs asks the upstream for its branches, tags and HEAD. With -// protocol v2 the server filters by prefix, which matters for Gerrit, where -// every patch set is a ref. Servers without v2 send every ref, and the rest -// are dropped here. -func (c *Cache) listUpstreamRefs(ctx context.Context, repoURL string) (remoteRefs, error) { - resp, err := c.upstreamRequest(ctx, http.MethodGet, repoURL+"/info/refs?service=git-upload-pack", "") - if err != nil { - return remoteRefs{}, err - } - defer func() { _ = resp.Body.Close() }() - if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, advertisementType) { - return remoteRefs{}, fmt.Errorf("upstream did not answer as a smart HTTP Git server (content type %q)", ct) - } - - pkts := newPktReader(io.LimitReader(resp.Body, maxRefsResponse)) - kind, line, err := pkts.next() - if err == nil && kind == pktData && strings.HasPrefix(line, "# service=") { - if kind, _, err = pkts.next(); err == nil && kind != pktFlushKind { - err = errors.New("no flush after the service line") - } - if err == nil { - kind, line, err = pkts.next() - } - } - if err != nil { - return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) - } - - if kind == pktData && line == "version 2" { - return c.lsRefs(ctx, repoURL) - } - if kind == pktData && line == "version 1" { - if kind, line, err = pkts.next(); err != nil { - return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) - } - } - return parseV0Advertisement(pkts, kind, line) -} - -// lsRefs runs the protocol v2 ls-refs command. -func (c *Cache) lsRefs(ctx context.Context, repoURL string) (remoteRefs, error) { - body := pktLine("command=ls-refs\n") + pktDelim + - pktLine("symrefs\n") + - pktLine("ref-prefix HEAD\n") + - pktLine("ref-prefix "+headsPrefix+"\n") + - pktLine("ref-prefix "+tagsPrefix+"\n") + - pktFlush - resp, err := c.upstreamRequest(ctx, http.MethodPost, repoURL+"/git-upload-pack", body) - if err != nil { - return remoteRefs{}, err - } - defer func() { _ = resp.Body.Close() }() - - remote := remoteRefs{refs: make(map[string]string)} - pkts := newPktReader(io.LimitReader(resp.Body, maxRefsResponse)) - for { - kind, line, err := pkts.next() - if err != nil { - return remoteRefs{}, fmt.Errorf("reading ls-refs response: %w", err) - } - if kind == pktFlushKind { - return remote, nil - } - if kind != pktData { - continue - } - fields := strings.Split(line, " ") - if len(fields) < 2 { //nolint:mnd // object id and ref name - return remoteRefs{}, fmt.Errorf("invalid ls-refs line %q", line) - } - if fields[1] == "HEAD" { - for _, attr := range fields[2:] { - if target, ok := strings.CutPrefix(attr, "symref-target:"); ok { - remote.head = target - } - } - continue - } - remote.add(fields[1], fields[0]) - } -} - -// parseV0Advertisement reads a protocol v0 ref advertisement whose first -// packet has already been read. The first ref carries the capabilities, -// including where HEAD points. -func parseV0Advertisement(pkts *pktReader, kind pktKind, line string) (remoteRefs, error) { - remote := remoteRefs{refs: make(map[string]string)} - first := true - for kind != pktFlushKind { - if kind == pktData { - ref, caps, _ := strings.Cut(line, "\x00") - if first { - remote.head = symrefTarget(caps, "HEAD") - first = false - } - oid, name, ok := strings.Cut(ref, " ") - if !ok { - return remoteRefs{}, fmt.Errorf("invalid ref advertisement line %q", ref) - } - remote.add(name, oid) - } - var err error - if kind, line, err = pkts.next(); err != nil { - return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) - } - } - return remote, nil -} - -// add keeps branches and tags, leaving out peeled tag entries. -func (r remoteRefs) add(name, oid string) { - if (strings.HasPrefix(name, headsPrefix) || strings.HasPrefix(name, tagsPrefix)) && !strings.HasSuffix(name, "^{}") { - r.refs[name] = oid - } -} - -func symrefTarget(caps, ref string) string { - for _, capability := range strings.Fields(caps) { - if value, ok := strings.CutPrefix(capability, "symref="+ref+":"); ok { - return value - } - } - return "" -} - -// upstreamRequest sends a smart HTTP request through the upstream client and -// classifies refusals. A non-empty body is sent as an upload-pack request. -func (c *Cache) upstreamRequest(ctx context.Context, method, target, body string) (*http.Response, error) { - var reader io.Reader - if body != "" { - reader = strings.NewReader(body) - } - req, err := http.NewRequestWithContext(ctx, method, target, reader) - if err != nil { - return nil, err - } - req.Header.Set("Git-Protocol", "version=2") - req.Header.Set("User-Agent", gitUserAgent) - if body != "" { - req.Header.Set("Content-Type", requestType) - req.Header.Set("Accept", resultType) - } - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("upstream request: %w", err) - } - if err := upstreamStatusError(resp.StatusCode); err != nil { - _ = resp.Body.Close() - return nil, err - } - return resp, nil -} - -func upstreamStatusError(status int) error { - switch { - case status == http.StatusOK: - return nil - case status == http.StatusNotFound || status == http.StatusGone: - return fmt.Errorf("%w: status %d", ErrNotFound, status) - case status == http.StatusUnauthorized || status == http.StatusForbidden: - return fmt.Errorf("%w: status %d", ErrRefused, status) - case status >= http.StatusMultipleChoices && status < http.StatusBadRequest: - return fmt.Errorf("%w: redirect (status %d) not followed", ErrRefused, status) - default: - return fmt.Errorf("upstream returned status %d", status) - } -} diff --git a/internal/gitcache/wants.go b/internal/gitcache/wants.go deleted file mode 100644 index 8755ba1..0000000 --- a/internal/gitcache/wants.go +++ /dev/null @@ -1,274 +0,0 @@ -package gitcache - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "maps" - "slices" - "strconv" - "strings" - "time" -) - -const ( - // wantedPrefix holds commits fetched because a client asked for them by - // object ID, such as a pull request head that is on no branch, as - // {wantedPrefix}{unix time}/{object ID}. The refs keep them from being - // garbage collected, and are hidden from clients. - wantedPrefix = "refs/proxy/wanted/" - // wantedMaxAge is how long a commit fetched by ID is kept while no - // cached branch or tag includes it. - wantedMaxAge = 7 * 24 * time.Hour - // wantRetryAfter is how long a want the upstream could not provide is - // not asked for again. Short, since a commit can reach a CI job before - // it reaches every upstream replica. A want too large for what - // max_repo_size leaves waits tooLargeRetryAfter instead. - wantRetryAfter = time.Minute - // maxFailedWants bounds the record of failed wants. - maxFailedWants = 4096 - // maxWantScan bounds how much of a request body is read for its wants. - maxWantScan = 1 << 20 - // maxWantFetch bounds the missing objects one request may have fetched. - maxWantFetch = 32 - // maxWantSplit bounds the wants of a failed fetch that are fetched again - // one at a time; a larger batch is refused as a whole. - maxWantSplit = 4 -) - -// readWants reads the object IDs a fetch request wants, from its first -// section, and returns them with a reader that replays the whole body. In -// protocol v0 that section holds the wants; in v2 it is the whole request. -func readWants(body io.Reader) ([]string, io.Reader) { - var consumed bytes.Buffer - pkts := newPktReader(io.TeeReader(io.LimitReader(body, maxWantScan), &consumed)) - var wants []string - for first := true; ; first = false { - kind, line, err := pkts.next() - if err != nil || kind == pktFlushKind { - break - } - if kind != pktData { - continue - } - if command, ok := strings.CutPrefix(line, "command="); ok && first && command != "fetch" { - break - } - if rest, ok := strings.CutPrefix(line, "want "); ok { - if oid, _, _ := strings.Cut(rest, " "); isObjectID(oid) { - wants = append(wants, oid) - } - } - } - return wants, io.MultiReader(&consumed, body) -} - -// fetchMissingWants fetches the wanted objects the repository lacks. Wants -// for branch and tag tips are already there after the refresh; this covers -// commits a client names directly, as actions/checkout does. -func (c *Cache) fetchMissingWants(ctx context.Context, rp *repo, repoURL string, wants []string) error { - missing, err := c.missingObjects(ctx, rp.dir, wants) - if err != nil { - return err - } - missing = c.withoutFailedWants(rp.key, missing) - if len(missing) == 0 || len(missing) > maxWantFetch { - return nil - } - unlock, err := rp.lockWrites(ctx) - if err != nil { - return err - } - defer unlock() - // Another request may have fetched them while this one waited. - if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil || len(missing) == 0 { - return err - } - defer c.evict() - - err = c.fetchWants(ctx, rp, repoURL, missing) - if err == nil || ctx.Err() != nil { - return err - } - if len(missing) == 1 || len(missing) > maxWantSplit || - errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) { - // Fetched one at a time, wants too large for what max_repo_size - // or max_size leaves would exhaust the same budget, and an upstream - // that did not answer would be waited on once per want. - c.recordFailedWants(rp.key, missing, err) - return err - } - // Find which wants failed, so a good one asked for with a bad one is - // not refused too. These fetches hold the repository's write lock, so - // only a small batch is split. - var errs []error - for _, oid := range missing { - if err := c.fetchWants(ctx, rp, repoURL, []string{oid}); err != nil { - if ctx.Err() != nil { - return err - } - c.recordFailedWants(rp.key, []string{oid}, err) - errs = append(errs, err) - } - } - return errors.Join(errs...) -} - -// fetchWants fetches oids from the upstream under wanted refs, within what -// max_repo_size leaves. A want that does not fit is refused, not the -// repository: transfer.unpackLimit makes a cut-off transfer leave nothing -// behind. -func (c *Cache) fetchWants(ctx context.Context, rp *repo, repoURL string, oids []string) error { - limit, err := c.fetchLimit(rp) - if err != nil { - return err - } - now := time.Now() - refspecs := make([]string, len(oids)) - for i, oid := range oids { - refspecs[i] = oid + ":" + wantedRef(oid, now) - } - res := c.newReservation() - err = c.fetch(ctx, rp.dir, repoURL, limit, res, []string{"--atomic"}, refspecs...) - res.settle(rp, dirSize(rp.dir)) - return err -} - -func wantedRef(oid string, at time.Time) string { - return wantedPrefix + strconv.FormatInt(at.Unix(), 10) + "/" + oid -} - -// pruneWanted deletes wanted refs that are no longer needed: those older -// than wantedMaxAge, and, after a fetch has moved branches or tags, those -// whose commit a cached branch or tag now includes. -func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fetched bool) error { - if len(wanted) == 0 { - return nil - } - cutoff := time.Now().Add(-wantedMaxAge).Unix() - var drop []string - keep := make(map[string]string, len(wanted)) - for _, ref := range wanted { - created, oid, ok := strings.Cut(strings.TrimPrefix(ref, wantedPrefix), "/") - unix, err := strconv.ParseInt(created, 10, 64) - if !ok || err != nil || unix < cutoff || !isObjectID(oid) { - drop = append(drop, ref) - continue - } - keep[oid] = ref - } - var mergedErr error - if fetched && len(keep) > 0 { - unmerged, err := c.unmergedCommits(ctx, dir, slices.Sorted(maps.Keys(keep))) - if err != nil { - mergedErr = fmt.Errorf("listing unmerged wanted commits: %w", err) - } else { - for oid, ref := range keep { - if !unmerged[oid] { - drop = append(drop, ref) - } - } - } - } - if len(drop) == 0 { - return mergedErr - } - var stdin strings.Builder - for _, ref := range drop { - stdin.WriteString("delete " + ref + "\n") - } - cmd := c.command(ctx, nil, "-C", dir, "update-ref", "--stdin") - cmd.Stdin = strings.NewReader(stdin.String()) - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("deleting wanted refs: %w: %s", err, strings.TrimSpace(string(out))) - } - return mergedErr -} - -// unmergedCommits returns which of commits no cached branch or tag -// includes. -func (c *Cache) unmergedCommits(ctx context.Context, dir string, commits []string) (map[string]bool, error) { - // rev-list prints what the commits, read from stdin, reach and no - // branch or tag does, so a commit it leaves out is merged. - cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") - cmd.Stdin = strings.NewReader(strings.Join(commits, "\n") + "\n") - out, err := cmd.Output() - if err != nil { - return nil, err - } - unmerged := make(map[string]bool) - for line := range strings.Lines(string(out)) { - unmerged[strings.TrimSpace(line)] = true - } - return unmerged, nil -} - -// withoutFailedWants drops the object IDs a recent fetch for the -// repository could not get. -func (c *Cache) withoutFailedWants(key string, oids []string) []string { - c.mu.Lock() - defer c.mu.Unlock() - now := time.Now() - return slices.DeleteFunc(oids, func(oid string) bool { - until, ok := c.failedWants[key+" "+oid] - return ok && now.Before(until) - }) -} - -// recordFailedWants keeps oids from being fetched again for the repository -// for wantRetryAfter, or for tooLargeRetryAfter when they did not fit in -// max_repo_size. -func (c *Cache) recordFailedWants(key string, oids []string, err error) { - now := time.Now() - until := now.Add(wantRetryAfter) - if errors.Is(err, ErrTooLarge) { - until = now.Add(tooLargeRetryAfter) - } - c.mu.Lock() - defer c.mu.Unlock() - for k, expiry := range c.failedWants { - if !now.Before(expiry) { - delete(c.failedWants, k) - } - } - for _, oid := range oids { - if len(c.failedWants) >= maxFailedWants { - return - } - c.failedWants[key+" "+oid] = until - } -} - -// missingObjects returns the object IDs in oids the repository does not have. -func (c *Cache) missingObjects(ctx context.Context, dir string, oids []string) ([]string, error) { - if len(oids) == 0 { - return nil, nil - } - cmd := c.command(ctx, nil, "-C", dir, "cat-file", "--batch-check") - cmd.Stdin = strings.NewReader(strings.Join(oids, "\n") + "\n") - out, err := cmd.Output() - if err != nil { - return nil, err - } - var missing []string - for line := range strings.Lines(string(out)) { - if oid, ok := strings.CutSuffix(strings.TrimSuffix(line, "\n"), " missing"); ok { - missing = append(missing, oid) - } - } - return missing, nil -} - -func isObjectID(s string) bool { - if len(s) != 40 && len(s) != 64 { //nolint:mnd // SHA-1 and SHA-256 hex lengths - return false - } - for _, r := range s { - if (r < '0' || r > '9') && (r < 'a' || r > 'f') { - return false - } - } - return true -} diff --git a/internal/gitcache/wants_test.go b/internal/gitcache/wants_test.go deleted file mode 100644 index f01df06..0000000 --- a/internal/gitcache/wants_test.go +++ /dev/null @@ -1,319 +0,0 @@ -package gitcache - -import ( - "crypto/rand" - "encoding/hex" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strconv" - "strings" - "testing" - "time" -) - -// pushPullRequest commits on a new branch from main, with data in a file if -// given, and pushes it only as refs/pull/{n}/head, so the commit is on no -// branch or tag upstream. -func (u *fakeUpstream) pushPullRequest(work string, n int, data []byte) string { - u.t.Helper() - branch := "pr" + strconv.Itoa(n) - runGit(u.t, work, "checkout", "--quiet", "-b", branch, "main") - if data != nil { - if err := os.WriteFile(filepath.Join(work, branch+".bin"), data, 0o600); err != nil { - u.t.Fatal(err) - } - runGit(u.t, work, "add", branch+".bin") - } - runGit(u.t, work, "commit", "--quiet", "--allow-empty", "-m", branch) - runGit(u.t, work, "push", "--quiet", "origin", "HEAD:refs/pull/"+strconv.Itoa(n)+"/head") - runGit(u.t, work, "checkout", "--quiet", "main") - return runGit(u.t, work, "rev-parse", branch) -} - -// fetchByID fetches one commit by ID from url, as actions/checkout does. -func fetchByID(t *testing.T, url, sha string) error { - t.Helper() - dir := t.TempDir() - runGit(t, dir, "init", "--quiet") - _, err := tryGit(t, dir, "fetch", "--quiet", "--depth=1", url, sha) - return err -} - -func randomBytes(n int) []byte { - b := make([]byte, n) - _, _ = rand.Read(b) - return b -} - -func (tc *testCache) wantedRefs(t *testing.T) []string { - t.Helper() - var refs []string - for line := range strings.Lines(tc.cachedRefs(t)) { - if _, name, _ := strings.Cut(strings.TrimSpace(line), " "); strings.HasPrefix(name, wantedPrefix) { - refs = append(refs, name) - } - } - return refs -} - -func TestMergedWantedCommitIsUnpinned(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - sha := upstream.pushPullRequest(work, 1, nil) - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - if err := fetchByID(t, tc.url("repo"), sha); err != nil { - t.Fatalf("fetch by ID: %v", err) - } - if refs := tc.wantedRefs(t); len(refs) != 1 { - t.Fatalf("wanted refs %v, want one", refs) - } - - runGit(t, work, "push", "--quiet", "origin", "pr1:main") - tc.clone(t, "repo") - if refs := tc.wantedRefs(t); len(refs) != 0 { - t.Errorf("wanted refs %v after the commit was merged, want none", refs) - } -} - -func TestOldWantedCommitIsUnpinned(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - old := upstream.pushPullRequest(work, 1, nil) - recent := upstream.pushPullRequest(work, 2, nil) - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - for _, sha := range []string{old, recent} { - if err := fetchByID(t, tc.url("repo"), sha); err != nil { - t.Fatalf("fetch by ID: %v", err) - } - } - - cached := filepath.Join(tc.dir, "up", "repo.git") - for _, ref := range tc.wantedRefs(t) { - if strings.HasSuffix(ref, "/"+old) { - runGit(t, cached, "update-ref", "-d", ref) - } - } - runGit(t, cached, "update-ref", wantedRef(old, time.Now().Add(-wantedMaxAge-time.Hour)), old) - - tc.clone(t, "repo") - refs := tc.wantedRefs(t) - if len(refs) != 1 || !strings.HasSuffix(refs[0], "/"+recent) { - t.Errorf("wanted refs %v, want only the recent commit's", refs) - } -} - -// TestRepositoryTooLargeToRefreshIsClonedAgain fills a repository with a -// commit fetched by ID, then grows its branch upstream past what is left of -// max_repo_size. Rather than stop following the upstream, the repository is -// dropped once idle and cloned again with only its branches and tags, which -// fit. -func TestRepositoryTooLargeToRefreshIsClonedAgain(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - pinned := upstream.pushPullRequest(work, 1, randomBytes(100<<10)) - base := repoSize(t, upstream, "repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 150<<10 }) - tc.clone(t, "repo") - if err := fetchByID(t, tc.url("repo"), pinned); err != nil { - t.Fatalf("fetch by ID: %v", err) - } - - if err := os.WriteFile(filepath.Join(work, "grown.bin"), randomBytes(80<<10), 0o600); err != nil { - t.Fatal(err) - } - runGit(t, work, "add", "grown.bin") - head := upstream.commit(work, "grown") - runGit(t, work, "push", "--quiet", "origin", "main") - - // This clone's refresh does not fit. It is served the cached copy, or, - // if the repository is dropped between its requests, the new clone. - tc.clone(t, "repo") - if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != head { - t.Errorf("HEAD = %s, want %s once cloned again", got, head) - } - waitIdle(t, tc.Cache) - if _, err := tryGit(t, filepath.Join(tc.dir, "up", "repo.git"), "cat-file", "-e", pinned); err == nil { - t.Error("the repository still holds the commit fetched by ID, so it was not cloned again") - } -} - -// TestTooLargeWantKeepsRepository asks for a commit by ID that does not fit -// in what max_repo_size leaves. The commit is refused, for a while, and the -// repository is kept, with nothing of the cut-off transfer left in it. -func TestTooLargeWantKeepsRepository(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - large := upstream.pushPullRequest(work, 1, randomBytes(200<<10)) - small := upstream.pushPullRequest(work, 2, nil) - base := repoSize(t, upstream, "repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 100<<10 }) - tc.clone(t, "repo") - - if err := fetchByID(t, tc.url("repo"), large); err == nil { - t.Fatal("fetched a commit larger than max_repo_size leaves") - } - waitIdle(t, tc.Cache) - cached := filepath.Join(tc.dir, "up", "repo.git") - if !fileExists(cached) { - t.Fatal("the repository was dropped") - } - if _, err := tryGit(t, cached, "cat-file", "-e", large); err == nil { - t.Error("the cut-off transfer left the commit behind") - } - if leftovers, _ := filepath.Glob(filepath.Join(cached, "objects", "pack", "tmp_*")); len(leftovers) != 0 { - t.Errorf("the cut-off transfer left %v", leftovers) - } - tc.mu.Lock() - until := tc.failedWants["up/repo.git "+large] - tc.mu.Unlock() - if time.Until(until) <= wantRetryAfter { - t.Errorf("the large commit is refused until %s, want for tooLargeRetryAfter", until) - } - - before := upstream.gitRequests.Load() - if err := fetchByID(t, tc.url("repo"), large); err == nil { - t.Fatal("fetched the large commit on a second try") - } - if got := upstream.gitRequests.Load() - before; got != 0 { - t.Errorf("the second try made %d upstream fetch requests, want none", got) - } - if err := fetchByID(t, tc.url("repo"), small); err != nil { - t.Errorf("fetch of another commit by ID: %v", err) - } -} - -// TestWantsFetchedTogetherFailSeparately asks for a good and a bad commit -// in one request. The good one is still fetched, and only the bad one is -// refused for a while. -func TestWantsFetchedTogetherFailSeparately(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - good := upstream.pushPullRequest(work, 1, nil) - bad := hex.EncodeToString(randomBytes(20)) - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - - dir := t.TempDir() - runGit(t, dir, "init", "--quiet") - if _, err := tryGit(t, dir, "fetch", "--quiet", tc.url("repo"), good, bad); err == nil { - t.Fatal("fetched a commit the upstream does not have") - } - waitIdle(t, tc.Cache) - if _, err := tryGit(t, filepath.Join(tc.dir, "up", "repo.git"), "cat-file", "-e", good); err != nil { - t.Error("the good commit was not fetched") - } - tc.mu.Lock() - _, goodRefused := tc.failedWants["up/repo.git "+good] - _, badRefused := tc.failedWants["up/repo.git "+bad] - tc.mu.Unlock() - if goodRefused || !badRefused { - t.Errorf("refused good=%v bad=%v, want only the bad commit", goodRefused, badRefused) - } -} - -func TestFailedWantIsNotRetriedAtOnce(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - missing := hex.EncodeToString(randomBytes(20)) - - before := upstream.gitRequests.Load() - if err := fetchByID(t, tc.url("repo"), missing); err == nil { - t.Fatal("fetched a commit the upstream does not have") - } - if upstream.gitRequests.Load() == before { - t.Fatal("the first request did not ask the upstream") - } - before = upstream.gitRequests.Load() - if err := fetchByID(t, tc.url("repo"), missing); err == nil { - t.Fatal("fetched a commit the upstream does not have") - } - if got := upstream.gitRequests.Load() - before; got != 0 { - t.Errorf("the second request made %d upstream fetch requests, want none", got) - } -} - -func TestWantsAreNotFetchedWhenTurnedOff(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - sha := upstream.pushPullRequest(work, 1, nil) - tc := newTestCache(t, upstream.server.URL, nil) - server := httptest.NewServer(testFront(tc.Cache, upstream.server.URL, false)) - defer server.Close() - url := server.URL + "/git/up/repo" - runGit(t, "", "clone", "--quiet", url, filepath.Join(t.TempDir(), "clone")) - - sessions := upstream.gitSessions.Load() - if err := fetchByID(t, url, sha); err == nil { - t.Error("fetched a commit by ID with FetchWants off") - } - if got := upstream.gitSessions.Load() - sessions; got != 0 { - t.Errorf("upstream saw %d fetches, want none", got) - } -} - -// TestWantsDuringAnOutageAreNotFetchedOneByOne asks for two commits while -// the upstream is down. One failed fetch records both, rather than a fetch -// per commit, each waiting on the upstream. -func TestWantsDuringAnOutageAreNotFetchedOneByOne(t *testing.T) { - upstream := newFakeUpstream(t) - work := upstream.newRepo("repo") - runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") - first := upstream.pushPullRequest(work, 1, nil) - second := upstream.pushPullRequest(work, 2, nil) - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - - upstream.status.Store(http.StatusServiceUnavailable) - sessions := upstream.gitSessions.Load() - dir := t.TempDir() - runGit(t, dir, "init", "--quiet") - if _, err := tryGit(t, dir, "fetch", "--quiet", tc.url("repo"), first, second); err == nil { - t.Fatal("fetched commits while the upstream was down") - } - if got := upstream.gitSessions.Load() - sessions; got != 1 { - t.Errorf("upstream saw %d fetches, want 1", got) - } - tc.mu.Lock() - _, firstRefused := tc.failedWants["up/repo.git "+first] - _, secondRefused := tc.failedWants["up/repo.git "+second] - tc.mu.Unlock() - if !firstRefused || !secondRefused { - t.Errorf("refused first=%v second=%v, want both", firstRefused, secondRefused) - } -} - -// TestLargeBatchOfFailedWantsIsNotSplit asks for more unknown commits at -// once than are fetched again one by one, which would hold the repository's -// write lock for a fetch per commit. -func TestLargeBatchOfFailedWantsIsNotSplit(t *testing.T) { - upstream := newFakeUpstream(t) - upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, nil) - tc.clone(t, "repo") - - args := []string{"fetch", "--quiet", tc.url("repo")} - for range maxWantSplit + 1 { - args = append(args, hex.EncodeToString(randomBytes(20))) - } - dir := t.TempDir() - runGit(t, dir, "init", "--quiet") - sessions := upstream.gitSessions.Load() - if _, err := tryGit(t, dir, args...); err == nil { - t.Fatal("fetched commits the upstream does not have") - } - if got := upstream.gitSessions.Load() - sessions; got != 1 { - t.Errorf("upstream saw %d fetches, want 1", got) - } -} diff --git a/internal/handler/apk.go b/internal/handler/apk.go deleted file mode 100644 index 045dbac..0000000 --- a/internal/handler/apk.go +++ /dev/null @@ -1,231 +0,0 @@ -package handler - -import ( - "crypto/sha256" - "encoding/hex" - "net/http" - "regexp" - "strings" - "time" - - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/purl" -) - -const ( - apkEcosystem = "alpine" - // defaultAPKRepositoryName is the repository name used when no - // upstream.apk repositories are configured. - defaultAPKRepositoryName = "alpine" - // defaultAPKUpstream is the official Alpine Linux mirror. - defaultAPKUpstream = "https://dl-cdn.alpinelinux.org/alpine" - apkMatchCount = 3 // full match + name + version -) - -// Packages are stored as pkg:apk/alpine/{name}, the namespace purl gives -// every apk package, whichever configured repository they came from. Package -// rules may leave the namespace out. -func init() { - retention.Default.Register(retention.Spec{Key: apkEcosystem, CanonicalPackage: apkRetentionKey}) -} - -// apkRetentionKey maps pkg:apk/{name} and pkg:apk/alpine/{name} to the stored -// package PURL. Another namespace, or a name with a slash, can never match a -// stored package and is refused. -func apkRetentionKey(p *purl.PURL) (string, bool) { - if p == nil || p.Type != "apk" || (p.Namespace != "" && p.Namespace != "alpine") || strings.Contains(p.Name, "/") { - return "", false - } - canonical := purl.MakePURLString(apkEcosystem, p.Name, "") - return canonical, canonical != "" -} - -// APKHandler handles Alpine APK repository protocol requests. Each configured -// upstream repository is mounted at /apk/{repository}/ and the remaining path -// mirrors the upstream layout ({release}/{repo}/{arch}/{file}). -// -// Repository indexes (v2 APKINDEX.tar.gz, v3 Packages.adb) and detached -// signatures are served byte-for-byte unchanged through the metadata cache so -// apk signature verification keeps working. Package files are cached in the -// shared artifact cache and stay available when the upstream is unreachable. -type APKHandler struct { - proxy *Proxy - proxyURL string - repositories map[string]string - // snapshots: see WithSnapshots. - snapshots time.Duration -} - -// NewAPKHandler creates an Alpine APK repository protocol handler. -// When repositories is empty, a single repository named "alpine" pointing at -// the official Alpine mirror is used. -func NewAPKHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *APKHandler { - return &APKHandler{ - proxy: proxy, - proxyURL: strings.TrimSuffix(proxyURL, "/"), - repositories: APKRepositories(repositories), - } -} - -// APKRepositories returns the upstream.apk repositories by name without -// trailing slashes, or the official Alpine mirror as "alpine" when none are -// configured. -func APKRepositories(configured map[string]string) map[string]string { - repositories := make(map[string]string, len(configured)) - for name, repositoryURL := range configured { - repositories[name] = strings.TrimSuffix(repositoryURL, "/") - } - if len(repositories) == 0 { - repositories[defaultAPKRepositoryName] = defaultAPKUpstream - } - return repositories -} - -// APKCacheFilename returns the filename a package at path -// ({release}/{repo}/{arch}/{file}) of repository is cached under. -func APKCacheFilename(repository, path string) string { - return repository + "/" + path -} - -// Routes returns the HTTP handler for APK requests. -// Mount this at /apk on your router. -func (h *APKHandler) Routes() http.Handler { - return rejectQueryChars(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet && r.Method != http.MethodHead { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - - path := strings.TrimPrefix(r.URL.Path, "/") - - if containsPathTraversal(path) { - http.Error(w, "invalid path", http.StatusBadRequest) - return - } - - segment, rest, ok := strings.Cut(path, "/") - repository, upstreamURL, day, snapshot, found := h.resolveRepository(segment) - if !ok || rest == "" || !found { - http.NotFound(w, r) - return - } - - switch { - case snapshot && (isAPKIndex(rest) || isAPKSignature(rest)): - // A dated snapshot serves the kept version of the index; packages - // and other files below are the same as under the plain path. - h.serveSnapshot(w, r, repository, upstreamURL, rest, day) - case isAPKIndex(rest) || isAPKSignature(rest): - // Indexes and detached signatures are signed upstream metadata. - // Cache them with the metadata TTL and serve the stored bytes - // unchanged so apk verification continues to work. - h.handleMetadata(w, r, repository, upstreamURL, rest) - case strings.HasSuffix(rest, ".apk"): - // Package downloads - cache these in the artifact cache. - h.handlePackageDownload(w, r, repository, upstreamURL, rest) - default: - // Other files - proxy directly. - h.proxyFile(w, r, upstreamURL, rest) - } - })) -} - -// isAPKIndex reports whether the path names a repository index: -// APKINDEX.tar.gz (apk v2) or Packages.adb (apk v3). -func isAPKIndex(path string) bool { - base := path[strings.LastIndex(path, "/")+1:] - return base == "APKINDEX.tar.gz" || base == "Packages.adb" -} - -// isAPKSignature reports whether the path names a detached signature file. -func isAPKSignature(path string) bool { - return strings.HasSuffix(path, ".sig") || strings.HasSuffix(path, ".rsa.pub") -} - -// handlePackageDownload fetches and caches .apk packages. -// Path format: {release}/{repo}/{arch}/{name}-{version}-r{rel}.apk -// Example: v3.22/main/x86_64/busybox-1.37.0-r12.apk -// -// APK filenames do not include the architecture, so the same filename can hold -// different bytes per architecture (and per release). The full request path is -// therefore part of the cache identity. -func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - name, version, arch := h.parseAPKPath(path) - if name == "" { - // Can't parse, just proxy directly - h.proxyFile(w, r, upstreamURL, path) - return - } - - downloadURL := upstreamURL + "/" + path - cacheFilename := APKCacheFilename(repository, path) - - h.proxy.Logger.Info("apk package download", - "repository", repository, "name", name, "version", version, "arch", arch) - - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), apkEcosystem, name, version, cacheFilename, downloadURL) - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") - return - } - - if result.Artifact.MediaType == "" { - result.Artifact.MediaType = "application/octet-stream" - } - ServeArtifactRequest(w, r, result) -} - -// handleMetadata serves repository indexes and signatures through the -// metadata cache. Stored bytes are re-served verbatim, which keeps embedded -// and detached signatures valid. -func (h *APKHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - cacheKey := h.metadataCacheKey(repository, upstreamURL, path) - if h.snapshotsEnabled() { - h.serveMetadataWithSnapshot(w, r, upstreamURL+"/"+path, cacheKey) - return - } - h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, apkEcosystem, cacheKey, "*/*") -} - -// metadataCacheKey derives the metadata cache key from the repository name, -// its upstream URL, and the request path. Hashing the identity keeps distinct -// repositories from sharing cache entries (repository names may contain '_' -// and a separator-based key would be ambiguous) and drops cached entries when -// a repository is repointed at a different upstream, mirroring -// HelmHandler.indexCacheKey. -func (h *APKHandler) metadataCacheKey(repository, upstreamURL, path string) string { - identity := repository + "\x00" + upstreamURL + "\x00" + path - digest := sha256.Sum256([]byte(identity)) - return hex.EncodeToString(digest[:]) -} - -// proxyFile proxies any file directly without caching. -func (h *APKHandler) proxyFile(w http.ResponseWriter, r *http.Request, upstreamURL, path string) { - h.proxy.ProxyFile(w, r, upstreamURL+"/"+path) -} - -// apkPackagePattern matches .apk filenames to extract name and version. -// Format: {name}-{version}-r{rel}.apk where version starts with a digit. -// Examples: -// - busybox-1.37.0-r12.apk -// - alpine-baselayout-data-3.7.0-r0.apk -var apkPackagePattern = regexp.MustCompile(`^(.+)-(\d[^-]*-r\d+)\.apk$`) - -// parseAPKPath extracts package info from a path containing an APK filename. -// The architecture is taken from the parent directory since APK filenames do -// not include it. -func (h *APKHandler) parseAPKPath(path string) (name, version, arch string) { - segments := strings.Split(path, "/") - filename := segments[len(segments)-1] - if len(segments) > 1 { - arch = segments[len(segments)-2] - } - - matches := apkPackagePattern.FindStringSubmatch(filename) - if len(matches) != apkMatchCount { - return "", "", "" - } - - return matches[1], matches[2], arch -} diff --git a/internal/handler/apk_snapshots.go b/internal/handler/apk_snapshots.go deleted file mode 100644 index 401dc00..0000000 --- a/internal/handler/apk_snapshots.go +++ /dev/null @@ -1,239 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "crypto/rand" - "crypto/sha256" - "database/sql" - "encoding/hex" - "io" - "net/http" - "strconv" - "strings" - "time" - - "github.com/git-pkgs/proxy/internal/database" -) - -// apkSnapshotDateLayout is the date format of a snapshot path segment. -const apkSnapshotDateLayout = "2006-01-02" - -// WithSnapshots keeps earlier versions of the indexes this handler serves and -// makes them available under /apk/{name}@{YYYY-MM-DD}/, as signed upstream -// bytes that apk verifies as usual. keep is how long a version is kept after -// it was replaced; 0 disables snapshots. -func (h *APKHandler) WithSnapshots(keep time.Duration) *APKHandler { - h.snapshots = keep - return h -} - -// snapshotsEnabled reports whether snapshots are recorded and served. They -// need the metadata cache, which stores the documents they are copied from. -func (h *APKHandler) snapshotsEnabled() bool { - return h.snapshots > 0 && h.proxy.CacheMetadata && h.proxy.DB != nil && h.proxy.Storage != nil -} - -// resolveRepository maps the first path segment to a configured repository. -// A segment of the form {name}@{YYYY-MM-DD} names a snapshot of repository -// name for that day, when snapshots are enabled and no repository is -// configured under the full segment. ok is false for anything else. -func (h *APKHandler) resolveRepository(segment string) (repository, upstreamURL string, day time.Time, snapshot, ok bool) { - if upstreamURL, found := h.repositories[segment]; found { - return segment, upstreamURL, time.Time{}, false, true - } - if !h.snapshotsEnabled() { - return "", "", time.Time{}, false, false - } - at := strings.LastIndexByte(segment, '@') - if at < 0 { - return "", "", time.Time{}, false, false - } - upstreamURL, found := h.repositories[segment[:at]] - if !found { - return "", "", time.Time{}, false, false - } - day, err := time.Parse(apkSnapshotDateLayout, segment[at+1:]) - if err != nil { - return "", "", time.Time{}, false, false - } - return segment[:at], upstreamURL, day, true, true -} - -// serveSnapshot answers a request for an index or signature under a snapshot -// path with the version that was current at the end of day. A day for which -// no version was stored, or one that has not begun yet, is 404. -func (h *APKHandler) serveSnapshot(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, day time.Time) { - end := day.AddDate(0, 0, 1) - if day.After(time.Now()) { - http.NotFound(w, r) - return - } - snap, err := h.proxy.DB.MetadataSnapshotAt(apkEcosystem, h.metadataCacheKey(repository, upstreamURL, path), end) - if err != nil { - h.proxy.Logger.Error("apk snapshot lookup failed", "repository", repository, "path", path, "error", err) - http.Error(w, "internal error", http.StatusInternalServerError) - return - } - if snap == nil { - http.NotFound(w, r) - return - } - - if writeMetadataValidators(w, r, cachedMeta{etag: `"` + snap.ContentDigest + `"`, lastModified: snap.ValidFrom()}) { - return - } - body, err := h.proxy.Storage.Open(r.Context(), snap.StoragePath) - if err != nil { - h.proxy.Logger.Error("apk snapshot unreadable", "repository", repository, "path", path, - "storage_path", snap.StoragePath, "error", err) - http.Error(w, "snapshot unavailable", http.StatusInternalServerError) - return - } - defer func() { _ = body.Close() }() - - if snap.ContentType.String != "" { - w.Header().Set(headerContentType, snap.ContentType.String) - } - w.Header().Set(headerContentLength, strconv.FormatInt(snap.Size, 10)) - if snap.ContentEncoding.String != "" { - w.Header().Set(headerContentEncoding, snap.ContentEncoding.String) - } - w.WriteHeader(http.StatusOK) - if r.Method != http.MethodHead { - _, _ = io.Copy(w, body) - } -} - -// serveMetadataWithSnapshot is ProxyCached, recording each version it serves. -func (h *APKHandler) serveMetadataWithSnapshot(w http.ResponseWriter, r *http.Request, upstream, cacheKey string) { - ctx := context.WithoutCancel(r.Context()) - h.proxy.proxyCachedWithEncoding(w, r, upstream, apkEcosystem, cacheKey, "identity", &metadataHooks{ - hit: func(entry *database.MetadataCacheEntry, seen time.Time) { - h.recordStoredSnapshot(ctx, cacheKey, entry, seen) - }, - fetched: func(body []byte, contentType, contentEncoding string) { - h.recordServedSnapshot(ctx, cacheKey, body, contentType, contentEncoding, time.Now()) - }, - }, "*/*") -} - -// recordServedSnapshot keeps body, the document a request is serving, as the -// version current from observed, unless the version current then already has -// its bytes. -func (h *APKHandler) recordServedSnapshot(ctx context.Context, cacheKey string, body []byte, contentType, contentEncoding string, observed time.Time) { - sum := sha256.Sum256(body) - digest := "sha256:" + hex.EncodeToString(sum[:]) - if !h.snapshotNeeded(cacheKey, digest, observed) { - return - } - storagePath := apkSnapshotStoragePath(cacheKey, digest, observed) - if _, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(body)); err != nil { - h.proxy.Logger.Warn("failed to store apk snapshot", "path", storagePath, "error", err) - h.discardSnapshotBlob(ctx, storagePath) - return - } - h.addSnapshot(ctx, &database.MetadataSnapshot{ - Ecosystem: apkEcosystem, - Name: cacheKey, - StoragePath: storagePath, - ContentType: sql.NullString{String: contentType, Valid: contentType != ""}, - ContentEncoding: sql.NullString{String: contentEncoding, Valid: contentEncoding != ""}, - ContentDigest: digest, - Size: int64(len(body)), - ValidFromMS: observed.UnixMilli(), - }) -} - -// recordStoredSnapshot keeps the cached document entry describes as the -// version current from observed, unless the version current then already -// has its digest. -func (h *APKHandler) recordStoredSnapshot(ctx context.Context, cacheKey string, entry *database.MetadataCacheEntry, observed time.Time) { - if !entry.ContentDigest.Valid || !h.snapshotNeeded(cacheKey, entry.ContentDigest.String, observed) { - return - } - h.storeSnapshot(ctx, cacheKey, entry, observed) -} - -// snapshotNeeded reports whether a version with digest seen at observed -// differs from the version current then. It is only a shortcut that saves -// copying the document: AddMetadataSnapshot makes the same check. -func (h *APKHandler) snapshotNeeded(cacheKey, digest string, observed time.Time) bool { - current, err := h.proxy.DB.MetadataSnapshotAt(apkEcosystem, cacheKey, observed.Add(time.Millisecond)) - return err == nil && (current == nil || current.ContentDigest != digest) -} - -// storeSnapshot records the cached document entry as the version current -// from validFrom. Concurrent requests can both get here for the same -// version; each copies to a path of its own, so the one whose row is not -// added only deletes its own copy. -func (h *APKHandler) storeSnapshot(ctx context.Context, cacheKey string, entry *database.MetadataCacheEntry, validFrom time.Time) { - digest := entry.ContentDigest.String - storagePath := apkSnapshotStoragePath(cacheKey, digest, validFrom) - size, ok := h.copySnapshotBlob(ctx, entry.StoragePath, storagePath, digest) - if !ok { - return - } - h.addSnapshot(ctx, &database.MetadataSnapshot{ - Ecosystem: apkEcosystem, - Name: cacheKey, - StoragePath: storagePath, - ContentType: entry.ContentType, - ContentEncoding: entry.ContentEncoding, - ContentDigest: digest, - Size: size, - ValidFromMS: validFrom.UnixMilli(), - }) -} - -// addSnapshot adds the row for a snapshot whose blob is stored, and deletes -// the blob when the row isn't added, as when another request recorded the -// same version first. -func (h *APKHandler) addSnapshot(ctx context.Context, snap *database.MetadataSnapshot) { - added, err := h.proxy.DB.AddMetadataSnapshot(snap) - if err != nil { - h.proxy.Logger.Warn("failed to record apk snapshot", "key", snap.Name, "error", err) - } - if err != nil || !added { - h.discardSnapshotBlob(ctx, snap.StoragePath) - } -} - -// copySnapshotBlob copies the cached document at from to to and reports its -// size, provided the copied bytes still have digest. A concurrent fetch can -// replace the cached document after its row was read; that version is -// recorded by the request that stored it. -func (h *APKHandler) copySnapshotBlob(ctx context.Context, from, to, digest string) (int64, bool) { - src, err := h.proxy.Storage.Open(ctx, from) - if err != nil { - return 0, false - } - defer func() { _ = src.Close() }() - size, hash, err := h.proxy.Storage.Store(ctx, to, src) - if err != nil { - h.proxy.Logger.Warn("failed to store apk snapshot", "path", to, "error", err) - return 0, false - } - if "sha256:"+hash != digest { - h.discardSnapshotBlob(ctx, to) - return 0, false - } - return size, true -} - -func (h *APKHandler) discardSnapshotBlob(ctx context.Context, path string) { - if err := h.proxy.Storage.Delete(ctx, path); err != nil { - h.proxy.Logger.Warn("failed to discard apk snapshot blob", "path", path, "error", err) - } -} - -// apkSnapshotStoragePath returns a new path to store one kept version of the -// document cached under cacheKey at. The time it became current keeps a -// version that returns after an upgrade was rolled back apart from its -// earlier copy, and a random suffix keeps two requests recording the same -// version in the same millisecond apart. -func apkSnapshotStoragePath(cacheKey, digest string, validFrom time.Time) string { - return "_snapshots/" + apkEcosystem + "/" + cacheKey + "/" + - strconv.FormatInt(validFrom.UnixMilli(), 10) + "-" + strings.TrimPrefix(digest, "sha256:") + - "-" + rand.Text() -} diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go deleted file mode 100644 index 83bb26d..0000000 --- a/internal/handler/apk_snapshots_test.go +++ /dev/null @@ -1,382 +0,0 @@ -package handler - -import ( - "io" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/registries/fetch" -) - -const apkSnapshotIndexPath = "/v3.22/main/x86_64/APKINDEX.tar.gz" - -// apkSnapshotUpstream serves the index held in index, initially "index v1", -// and counts package downloads. -type apkSnapshotUpstream struct { - index atomic.Value - downloads atomic.Int32 - server *httptest.Server -} - -func newAPKSnapshotUpstream(t *testing.T) *apkSnapshotUpstream { - t.Helper() - u := &apkSnapshotUpstream{} - u.index.Store("index v1") - u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case apkSnapshotIndexPath: - _, _ = w.Write([]byte(u.index.Load().(string))) - case "/v3.22/main/x86_64/curl-8.5.0-r0.apk": - u.downloads.Add(1) - _, _ = w.Write([]byte("curl 8.5.0")) - default: - http.NotFound(w, r) - } - })) - t.Cleanup(u.server.Close) - return u -} - -// newSnapshotAPKHandler returns an APK handler with snapshots enabled whose -// metadata TTL is 0, so every index request asks upstream. -func newSnapshotAPKHandler(t *testing.T, upstreamURL string) (*APKHandler, *database.DB) { - t.Helper() - proxy, db, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = 0 - proxy.HTTPClient = http.DefaultClient - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(http.DefaultClient), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewAPKHandler(proxy, "http://proxy.example", map[string]string{"alpine": upstreamURL}). - WithSnapshots(30 * 24 * time.Hour) - return h, db -} - -func snapshotCount(t *testing.T, db *database.DB) int { - t.Helper() - var n int - if err := db.Get(&n, `SELECT COUNT(*) FROM metadata_snapshots`); err != nil { - t.Fatal(err) - } - return n -} - -func expectAPKResponse(t *testing.T, h *APKHandler, target string, status int, body string) { - t.Helper() - w := serveAPKRequest(h, target) - if w.Code != status { - t.Fatalf("%s: status = %d, want %d: %s", target, w.Code, status, w.Body.String()) - } - if status == http.StatusOK && w.Body.String() != body { - t.Errorf("%s: body = %q, want %q", target, w.Body.String(), body) - } -} - -// TestAPKSnapshots_ServeIndexCurrentAtDate records two versions of an index -// through the plain path and checks that each dated path serves the version -// that was current at the end of that day. -func TestAPKSnapshots_ServeIndexCurrentAtDate(t *testing.T) { - upstream := newAPKSnapshotUpstream(t) - h, db := newSnapshotAPKHandler(t, upstream.server.URL) - - expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") - // Pretend v1 was first stored on 2026-01-10. - recorded := time.Date(2026, 1, 10, 12, 0, 0, 0, time.UTC) - if _, err := db.Exec(`UPDATE metadata_snapshots SET valid_from_ms = ?`, recorded.UnixMilli()); err != nil { - t.Fatal(err) - } - - upstream.index.Store("index v2") - expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v2") - // An unchanged index adds no snapshot. - expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v2") - if got := snapshotCount(t, db); got != 2 { - t.Fatalf("snapshots = %d, want 2", got) - } - - today := time.Now().UTC().Format(apkSnapshotDateLayout) - future := time.Now().UTC().AddDate(0, 0, 2).Format(apkSnapshotDateLayout) - expectAPKResponse(t, h, "/alpine@2026-01-09"+apkSnapshotIndexPath, http.StatusNotFound, "") - expectAPKResponse(t, h, "/alpine@2026-01-10"+apkSnapshotIndexPath, http.StatusOK, "index v1") - expectAPKResponse(t, h, "/alpine@"+today+apkSnapshotIndexPath, http.StatusOK, "index v2") - expectAPKResponse(t, h, "/alpine@"+future+apkSnapshotIndexPath, http.StatusNotFound, "") - expectAPKResponse(t, h, "/alpine@2026-13-01"+apkSnapshotIndexPath, http.StatusNotFound, "") - expectAPKResponse(t, h, "/other@2026-01-10"+apkSnapshotIndexPath, http.StatusNotFound, "") - expectAPKResponse(t, h, "/alpine@2026-01-10/v3.22/community/x86_64/APKINDEX.tar.gz", http.StatusNotFound, "") -} - -func TestAPKSnapshots_ConditionalRequest(t *testing.T) { - upstream := newAPKSnapshotUpstream(t) - h, _ := newSnapshotAPKHandler(t, upstream.server.URL) - expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") - - target := "/alpine@" + time.Now().UTC().Format(apkSnapshotDateLayout) + apkSnapshotIndexPath - first := serveAPKRequest(h, target) - etag := first.Header().Get("ETag") - if first.Code != http.StatusOK || etag == "" { - t.Fatalf("status = %d, ETag = %q, want 200 with an ETag", first.Code, etag) - } - req := httptest.NewRequest(http.MethodGet, target, nil) - req.Header.Set("If-None-Match", etag) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusNotModified { - t.Errorf("conditional request: status = %d, want 304", w.Code) - } -} - -// TestAPKSnapshots_PackagesShareTheCache checks that a package requested -// under a snapshot path is the cached artifact of the plain path. -func TestAPKSnapshots_PackagesShareTheCache(t *testing.T) { - upstream := newAPKSnapshotUpstream(t) - h, _ := newSnapshotAPKHandler(t, upstream.server.URL) - - pkg := "/v3.22/main/x86_64/curl-8.5.0-r0.apk" - expectAPKResponse(t, h, "/alpine"+pkg, http.StatusOK, "curl 8.5.0") - expectAPKResponse(t, h, "/alpine@2026-01-10"+pkg, http.StatusOK, "curl 8.5.0") - if got := upstream.downloads.Load(); got != 1 { - t.Errorf("upstream downloads = %d, want 1", got) - } - expectAPKResponse(t, h, "/alpine@2026-01-10/v3.22/main/x86_64/gone-1.0-r0.apk", http.StatusNotFound, "") -} - -func TestAPKSnapshots_DisabledByDefault(t *testing.T) { - upstream := newAPKSnapshotUpstream(t) - h, db := newSnapshotAPKHandler(t, upstream.server.URL) - h.WithSnapshots(0) - - expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") - if got := snapshotCount(t, db); got != 0 { - t.Errorf("snapshots = %d, want 0", got) - } - today := time.Now().UTC().Format(apkSnapshotDateLayout) - expectAPKResponse(t, h, "/alpine@"+today+apkSnapshotIndexPath, http.StatusNotFound, "") -} - -// TestAPKSnapshots_ConfiguredNameWins checks that a repository whose -// configured name contains '@' is served as configured, not as a snapshot. -func TestAPKSnapshots_ConfiguredNameWins(t *testing.T) { - upstream := newAPKSnapshotUpstream(t) - h, _ := newSnapshotAPKHandler(t, upstream.server.URL) - h.repositories["alpine@2026-01-10"] = upstream.server.URL - - expectAPKResponse(t, h, "/alpine@2026-01-10"+apkSnapshotIndexPath, http.StatusOK, "index v1") -} - -// TestAPKSnapshots_SameVersionRecordedTwice checks that two requests -// recording the same version in the same millisecond, as coalesced fetches -// after an upstream update can, leave the recorded snapshot readable: the -// request whose row is not added must not delete the other's blob. -func TestAPKSnapshots_SameVersionRecordedTwice(t *testing.T) { - u := newAPKSnapshotUpstream(t) - h, db := newSnapshotAPKHandler(t, u.server.URL) - expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") - - key := h.metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) - entry, err := db.GetMetadataCache(apkEcosystem, key) - if err != nil || entry == nil { - t.Fatalf("cached index row = %v, %v", entry, err) - } - if _, err := db.Exec(`DELETE FROM metadata_snapshots`); err != nil { - t.Fatal(err) - } - validFrom := time.Now() - h.storeSnapshot(t.Context(), key, entry, validFrom) - h.storeSnapshot(t.Context(), key, entry, validFrom) - - if n := snapshotCount(t, db); n != 1 { - t.Fatalf("%d snapshots recorded, want 1", n) - } - snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) - if err != nil || snap == nil { - t.Fatalf("latest snapshot = %v, %v", snap, err) - } - body, err := h.proxy.Storage.Open(t.Context(), snap.StoragePath) - if err != nil { - t.Fatalf("recorded snapshot blob unreadable: %v", err) - } - _ = body.Close() - day := validFrom.UTC().Format(apkSnapshotDateLayout) - expectAPKResponse(t, h, "/alpine@"+day+apkSnapshotIndexPath, http.StatusOK, "index v1") -} - -// pausingWriter blocks the first Write until release is closed, after -// signalling on writing. -type pausingWriter struct { - *httptest.ResponseRecorder - once sync.Once - writing chan struct{} - release chan struct{} -} - -func (w *pausingWriter) Write(b []byte) (int, error) { - w.once.Do(func() { - close(w.writing) - <-w.release - }) - return w.ResponseRecorder.Write(b) -} - -// TestAPKSnapshots_OverlappingRequests pauses a request while it writes -// index v1, lets a second request fetch and serve v2, and then lets the first -// one finish. Both versions must be kept, in the order they were fetched, -// with readable blobs. -func TestAPKSnapshots_OverlappingRequests(t *testing.T) { - u := newAPKSnapshotUpstream(t) - h, db := newSnapshotAPKHandler(t, u.server.URL) - target := "/alpine" + apkSnapshotIndexPath - - first := &pausingWriter{ - ResponseRecorder: httptest.NewRecorder(), - writing: make(chan struct{}), - release: make(chan struct{}), - } - done := make(chan struct{}) - go func() { - defer close(done) - h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, target, nil)) - }() - select { - case <-first.writing: - case <-done: - t.Fatal("first request finished without writing a body") - } - - u.index.Store("index v2") - expectAPKResponse(t, h, target, http.StatusOK, "index v2") - close(first.release) - <-done - if first.Code != http.StatusOK || first.Body.String() != "index v1" { - t.Fatalf("first request = %d %q, want 200 with index v1", first.Code, first.Body.String()) - } - - var snaps []database.MetadataSnapshot - if err := db.Select(&snaps, `SELECT `+ - `id, ecosystem, name, storage_path, content_type, content_encoding, content_digest, size, valid_from_ms `+ - `FROM metadata_snapshots ORDER BY valid_from_ms, id`); err != nil { - t.Fatal(err) - } - if len(snaps) != 2 { - t.Fatalf("%d snapshots recorded, want 2", len(snaps)) - } - for i, want := range []string{"index v1", "index v2"} { - body, err := h.proxy.Storage.Open(t.Context(), snaps[i].StoragePath) - if err != nil { - t.Fatalf("snapshot %d blob unreadable: %v", i, err) - } - got, _ := io.ReadAll(body) - _ = body.Close() - if string(got) != want { - t.Errorf("snapshot %d = %q, want %q", i, got, want) - } - if snaps[i].Size != int64(len(want)) { - t.Errorf("snapshot %d size = %d, want %d", i, snaps[i].Size, len(want)) - } - } -} - -// TestAPKSnapshots_ConcurrentReplicas records the same index version from -// two replicas sharing a Postgres database, many times at once. One snapshot -// row must remain, with a readable blob, and the other copies are deleted. -func TestAPKSnapshots_ConcurrentReplicas(t *testing.T) { - a, b, db := sharedInstances(t) - u := newAPKSnapshotUpstream(t) - handlers := make([]*APKHandler, 0, 2) - for _, p := range []*Proxy{a, b} { - p.CacheMetadata = true - p.HTTPClient = http.DefaultClient - handlers = append(handlers, NewAPKHandler(p, "http://proxy.example", map[string]string{"alpine": u.server.URL}). - WithSnapshots(30*24*time.Hour)) - } - expectAPKResponse(t, handlers[0], "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") - - key := handlers[0].metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) - entry, err := db.GetMetadataCache(apkEcosystem, key) - if err != nil || entry == nil { - t.Fatalf("cached index row = %v, %v", entry, err) - } - if _, err := db.Exec(`DELETE FROM metadata_snapshots`); err != nil { - t.Fatal(err) - } - // The request above recorded a copy of its own, which the test leaves be. - store := a.Storage.(*mockStorage) - store.mu.Lock() - before := map[string]bool{} - for path := range store.files { - before[path] = true - } - store.mu.Unlock() - - validFrom := time.Now() - var wg sync.WaitGroup - for i := range 16 { - wg.Go(func() { handlers[i%2].storeSnapshot(t.Context(), key, entry, validFrom) }) - } - wg.Wait() - - if n := snapshotCount(t, db); n != 1 { - t.Fatalf("%d snapshots recorded, want 1", n) - } - snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) - if err != nil || snap == nil { - t.Fatalf("latest snapshot = %v, %v", snap, err) - } - store.mu.Lock() - defer store.mu.Unlock() - if got := string(store.files[snap.StoragePath]); got != "index v1" { - t.Errorf("snapshot blob = %q, want the index", got) - } - for path := range store.files { - if strings.HasPrefix(path, "_snapshots/") && path != snap.StoragePath && !before[path] { - t.Errorf("copy %s of a snapshot that wasn't recorded is left in storage", path) - } - } -} - -// TestAPKSnapshots_CacheHit records an index cached before snapshots were -// enabled when a request is served from the cache within its TTL, and adds -// no row for a later hit on the same version. -func TestAPKSnapshots_CacheHit(t *testing.T) { - u := newAPKSnapshotUpstream(t) - h, db := newSnapshotAPKHandler(t, u.server.URL) - h.proxy.MetadataTTL = 5 * time.Minute - h.WithSnapshots(0) - expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") - if n := snapshotCount(t, db); n != 0 { - t.Fatalf("%d snapshots recorded with snapshots off, want 0", n) - } - - h.WithSnapshots(30 * 24 * time.Hour) - // Upstream moved on; a hit still serves the cached v1. - u.index.Store("index v2") - expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") - if n := snapshotCount(t, db); n != 1 { - t.Fatalf("%d snapshots recorded on a cache hit, want 1", n) - } - key := h.metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) - snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) - if err != nil || snap == nil { - t.Fatalf("latest snapshot = %v, %v", snap, err) - } - body, err := h.proxy.Storage.Open(t.Context(), snap.StoragePath) - if err != nil { - t.Fatalf("recorded snapshot blob unreadable: %v", err) - } - got, _ := io.ReadAll(body) - _ = body.Close() - if string(got) != "index v1" { - t.Fatalf("snapshot blob = %q, want %q", got, "index v1") - } - - expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") - if n := snapshotCount(t, db); n != 1 { - t.Fatalf("%d snapshots after a second hit on the same version, want 1", n) - } -} diff --git a/internal/handler/apk_test.go b/internal/handler/apk_test.go deleted file mode 100644 index fc90681..0000000 --- a/internal/handler/apk_test.go +++ /dev/null @@ -1,390 +0,0 @@ -package handler - -import ( - "fmt" - "net/http" - "net/http/httptest" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries/fetch" -) - -func TestAPKHandler_parseAPKPath(t *testing.T) { - h := &APKHandler{} - - assertPathParser(t, "parseAPKPath", h.parseAPKPath, []pathParseCase{ - {"v3.22/main/x86_64/busybox-1.37.0-r12.apk", "busybox", "1.37.0-r12", "x86_64"}, - {"v3.22/main/aarch64/alpine-baselayout-data-3.7.0-r0.apk", "alpine-baselayout-data", "3.7.0-r0", "aarch64"}, - {"edge/community/x86_64/openjdk21-jre-21.0.2_p13-r1.apk", "openjdk21-jre", "21.0.2_p13-r1", "x86_64"}, - {"busybox-1.37.0-r12.apk", "busybox", "1.37.0-r12", ""}, - {"v3.22/main/x86_64/invalid.apk", "", "", ""}, - {"v3.22/main/x86_64/not-an-apk-file", "", "", ""}, - }) -} - -func TestAPKHandler_Routes(t *testing.T) { - h := NewAPKHandler(nil, "http://localhost:8080", nil) - assertRoutesBasics(t, h.Routes(), "/alpine/v3.22/main/x86_64/APKINDEX.tar.gz", "/alpine/v3.22/../../../etc/passwd") -} - -func TestAPKHandler_DefaultsToOfficialMirror(t *testing.T) { - h := NewAPKHandler(nil, "http://localhost:8080", nil) - if got := h.repositories[defaultAPKRepositoryName]; got != defaultAPKUpstream { - t.Errorf("default repository = %q, want %q", got, defaultAPKUpstream) - } -} - -func TestAPKHandler_UnknownRepositoryReturns404(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - h := NewAPKHandler(proxy, "http://localhost:8080", map[string]string{"alpine": "https://example.test"}) - - for _, target := range []string{ - "/unknown/v3.22/main/x86_64/APKINDEX.tar.gz", - "/alpine", - "/", - } { - w := serveAPKRequest(h, target) - if w.Code != http.StatusNotFound { - t.Errorf("%s: status = %d, want 404", target, w.Code) - } - } -} - -// TestAPKHandler_MetadataCacheKeysDoNotCollideAcrossRepositories guards the -// hashed metadata cache key: with a separator-based key, repositories named -// "alpine" and "alpine_edge" would share cache entries for -// /alpine/edge/main/x86_64/APKINDEX.tar.gz and -// /alpine_edge/main/x86_64/APKINDEX.tar.gz, serving one repository's signed -// index to clients of the other. -func TestAPKHandler_MetadataCacheKeysDoNotCollideAcrossRepositories(t *testing.T) { - indexA := "signed index of repository A" - upstreamA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/edge/main/x86_64/APKINDEX.tar.gz" { - http.NotFound(w, r) - return - } - _, _ = fmt.Fprint(w, indexA) - })) - defer upstreamA.Close() - - indexB := "signed index of repository B" - upstreamB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/main/x86_64/APKINDEX.tar.gz" { - http.NotFound(w, r) - return - } - _, _ = fmt.Fprint(w, indexB) - })) - defer upstreamB.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = http.DefaultClient - - h := NewAPKHandler(proxy, "http://proxy.example", map[string]string{ - "alpine": upstreamA.URL, - "alpine_edge": upstreamB.URL, - }) - - first := serveAPKRequest(h, "/alpine/edge/main/x86_64/APKINDEX.tar.gz") - if first.Code != http.StatusOK || first.Body.String() != indexA { - t.Fatalf("repository A: status = %d, body = %q, want 200 %q", first.Code, first.Body.String(), indexA) - } - - // Served within the metadata TTL: a colliding key would return indexA here. - second := serveAPKRequest(h, "/alpine_edge/main/x86_64/APKINDEX.tar.gz") - if second.Code != http.StatusOK { - t.Fatalf("repository B: status = %d, want 200: %s", second.Code, second.Body.String()) - } - if second.Body.String() != indexB { - t.Errorf("repository B served %q, want %q (cache key collision)", second.Body.String(), indexB) - } -} - -// TestAPKHandler_IndexesServedUnchanged covers v2 (APKINDEX.tar.gz) and v3 -// (Packages.adb) indexes plus detached signatures: bytes must be served -// unchanged so apk signature verification keeps working, and within the -// metadata TTL cached copies must be served without contacting the upstream -// (the stale-after-TTL fallback itself is covered by the shared ProxyCached -// tests). -func TestAPKHandler_IndexesServedUnchanged(t *testing.T) { - files := map[string][]byte{ - "/v3.22/main/x86_64/APKINDEX.tar.gz": []byte("\x1f\x8b\x08v2-index-with-embedded-signature"), - "/v3.22/main/x86_64/Packages.adb": []byte("ADB.v3-index-binary\x00payload"), - "/v3.22/main/x86_64/Packages.adb.sig": []byte("detached-signature-bytes"), - } - - var available atomic.Bool - available.Store(true) - var upstreamRequests atomic.Int32 - var authHeader string - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - authHeader = r.Header.Get("Authorization") - data, ok := files[r.URL.Path] - if !ok { - http.NotFound(w, r) - return - } - upstreamRequests.Add(1) - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = w.Write(data) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - authorizeUpstream(proxy, "Bearer apk-token") - - h := NewAPKHandler(proxy, "http://proxy.example", map[string]string{"alpine": upstream.URL}) - - for path, want := range files { - w := serveAPKRequest(h, "/alpine"+path) - if w.Code != http.StatusOK { - t.Fatalf("%s: status = %d, want 200: %s", path, w.Code, w.Body.String()) - } - if got := w.Body.Bytes(); string(got) != string(want) { - t.Errorf("%s: body altered:\ngot %q\nwant %q", path, got, want) - } - } - if authHeader != "Bearer apk-token" { - t.Errorf("Authorization = %q, want %q", authHeader, "Bearer apk-token") - } - - // Upstream goes away: cached indexes must still be served, unchanged. - available.Store(false) - requestsBefore := upstreamRequests.Load() - for path, want := range files { - w := serveAPKRequest(h, "/alpine"+path) - if w.Code != http.StatusOK { - t.Fatalf("%s offline: status = %d, want 200: %s", path, w.Code, w.Body.String()) - } - if got := w.Body.Bytes(); string(got) != string(want) { - t.Errorf("%s offline: body altered:\ngot %q\nwant %q", path, got, want) - } - } - if got := upstreamRequests.Load(); got != requestsBefore { - t.Errorf("upstream requests during offline reads = %d, want %d", got, requestsBefore) - } -} - -// TestAPKHandler_PackageDownloadCachesPerArch covers package downloads, cache -// hits, offline reads, and that identically named packages for different -// architectures are cached separately. -func TestAPKHandler_PackageDownloadCachesPerArch(t *testing.T) { - packages := map[string][]byte{ - "/v3.22/main/x86_64/busybox-1.37.0-r12.apk": []byte("x86_64 package bytes"), - "/v3.22/main/aarch64/busybox-1.37.0-r12.apk": []byte("aarch64 package bytes"), - } - - var available atomic.Bool - available.Store(true) - var packageRequests atomic.Int32 - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - data, ok := packages[r.URL.Path] - if !ok { - http.NotFound(w, r) - return - } - packageRequests.Add(1) - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = w.Write(data) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewAPKHandler(proxy, "http://proxy.example", map[string]string{"alpine": upstream.URL}) - - for path, want := range packages { - w := serveAPKRequest(h, "/alpine"+path) - if w.Code != http.StatusOK { - t.Fatalf("%s: status = %d, want 200: %s", path, w.Code, w.Body.String()) - } - if got := w.Body.String(); got != string(want) { - t.Errorf("%s: body = %q, want %q", path, got, want) - } - } - if got := packageRequests.Load(); got != 2 { - t.Fatalf("upstream package requests = %d, want 2 (one per architecture)", got) - } - - // Second round must be served from cache, even with the upstream down. - available.Store(false) - for path, want := range packages { - w := serveAPKRequest(h, "/alpine"+path) - if w.Code != http.StatusOK { - t.Fatalf("%s cached: status = %d, want 200: %s", path, w.Code, w.Body.String()) - } - if got := w.Body.String(); got != string(want) { - t.Errorf("%s cached: body = %q, want %q", path, got, want) - } - } - if got := packageRequests.Load(); got != 2 { - t.Errorf("upstream package requests after cache hits = %d, want 2", got) - } -} - -func TestAPKHandler_PackageDownloadSendsUpstreamAuth(t *testing.T) { - var authHeader string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - authHeader = r.Header.Get("Authorization") - if authHeader != "Bearer apk-token" { - w.WriteHeader(http.StatusUnauthorized) - return - } - _, _ = fmt.Fprint(w, "private package") - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - client := upstream.Client() - client.Transport = &authRoundTripper{base: client.Transport, header: "Authorization", value: "Bearer apk-token"} - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewAPKHandler(proxy, "http://proxy.example", map[string]string{"private": upstream.URL}) - - w := serveAPKRequest(h, "/private/v3.22/main/x86_64/busybox-1.37.0-r12.apk") - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - if w.Body.String() != "private package" { - t.Errorf("body = %q, want %q", w.Body.String(), "private package") - } - if authHeader != "Bearer apk-token" { - t.Errorf("Authorization = %q, want %q", authHeader, "Bearer apk-token") - } -} - -func TestAPKHandler_UnparseablePackageProxiedDirectly(t *testing.T) { - var requested string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requested = r.URL.Path - _, _ = fmt.Fprint(w, "raw bytes") - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - - h := NewAPKHandler(proxy, "http://proxy.example", map[string]string{"alpine": upstream.URL}) - - w := serveAPKRequest(h, "/alpine/v3.22/main/x86_64/no-version.apk") - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - if requested != "/v3.22/main/x86_64/no-version.apk" { - t.Errorf("upstream path = %q, want %q", requested, "/v3.22/main/x86_64/no-version.apk") - } - if w.Body.String() != "raw bytes" { - t.Errorf("body = %q, want %q", w.Body.String(), "raw bytes") - } -} - -// authRoundTripper adds a static auth header, mimicking the server's -// authentication-aware upstream transport. -type authRoundTripper struct { - base http.RoundTripper - header string - value string -} - -func (a *authRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - req = req.Clone(req.Context()) - req.Header.Set(a.header, a.value) - base := a.base - if base == nil { - base = http.DefaultTransport - } - return base.RoundTrip(req) -} - -// TestAPKHandler_PackageHeadOmitsBody verifies that HEAD requests for cached -// packages return headers (including Content-Length) without a body. -func TestAPKHandler_PackageHeadOmitsBody(t *testing.T) { - pkg := []byte("package bytes") - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = w.Write(pkg) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewAPKHandler(proxy, "http://proxy.example", map[string]string{"alpine": upstream.URL}) - - target := "/alpine/v3.22/main/x86_64/busybox-1.37.0-r12.apk" - if w := serveAPKRequest(h, target); w.Code != http.StatusOK { - t.Fatalf("seeding GET: status = %d, want 200: %s", w.Code, w.Body.String()) - } - - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, target, nil)) - if w.Code != http.StatusOK { - t.Fatalf("HEAD: status = %d, want 200: %s", w.Code, w.Body.String()) - } - if got := w.Body.Len(); got != 0 { - t.Errorf("HEAD body length = %d, want 0", got) - } - if got := w.Header().Get("Content-Length"); got != fmt.Sprint(len(pkg)) { - t.Errorf("HEAD Content-Length = %q, want %d", got, len(pkg)) - } -} - -func serveAPKRequest(h *APKHandler, target string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - return w -} - -func TestAPKRetentionKey(t *testing.T) { - tests := []struct { - key string - want string - ok bool - }{ - {"pkg:apk/curl", "pkg:apk/alpine/curl", true}, - {"pkg:apk/alpine/curl", "pkg:apk/alpine/curl", true}, - {"pkg:apk/Alpine/Curl", "pkg:apk/alpine/curl", true}, - {"pkg:apk/alpine/libstdc++", "pkg:apk/alpine/libstdc%2B%2B", true}, - {"pkg:apk/wolfi/curl", "", false}, - {"pkg:apk/alpine/main%2Fcurl", "", false}, - {"pkg:deb/alpine/curl", "", false}, - } - for _, tt := range tests { - p, err := purl.Parse(tt.key) - if err != nil { - t.Fatalf("Parse(%s): %v", tt.key, err) - } - got, ok := apkRetentionKey(p) - if ok != tt.ok || (ok && got != tt.want) { - t.Errorf("apkRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) - } - } - if _, ok := apkRetentionKey(nil); ok { - t.Error("apkRetentionKey(nil) accepted") - } -} diff --git a/internal/handler/arch.go b/internal/handler/arch.go deleted file mode 100644 index b53f25c..0000000 --- a/internal/handler/arch.go +++ /dev/null @@ -1,507 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha256" - "database/sql" - "encoding/hex" - "errors" - "fmt" - "net/http" - "regexp" - "strings" - "sync" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/metrics" -) - -const ( - archEcosystem = "arch" - defaultArchRepositoryName = "archlinux" - // defaultArchUpstream is the official Arch Linux GeoIP mirror. - defaultArchUpstream = "https://geo.mirror.pkgbuild.com" - archMatchCount = 4 // full match + name + version + arch - archSignatureSuffix = ".sig" - archAccept = "*/*" - archPackageMediaType = "application/octet-stream" -) - -// ArchHandler serves pacman repositories at /arch/{repository}/. The rest of -// the path goes to the mirror unchanged, so the Arch Linux ARM layout works -// too. Databases are served byte for byte through the metadata cache, packages -// and their signatures through the artifact cache. -type ArchHandler struct { - proxy *Proxy - proxyURL string - repositories map[string]string - locks sync.Map // database cache key -> *sync.Mutex -} - -// NewArchHandler creates an Arch Linux handler. Without repositories it serves -// the official GeoIP mirror as "archlinux". -func NewArchHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *ArchHandler { - h := &ArchHandler{ - proxy: proxy, - proxyURL: strings.TrimSuffix(proxyURL, "/"), - repositories: make(map[string]string, len(repositories)), - } - for name, repositoryURL := range repositories { - h.repositories[name] = strings.TrimSuffix(repositoryURL, "/") - } - if len(h.repositories) == 0 { - h.repositories[defaultArchRepositoryName] = defaultArchUpstream - } - return h -} - -// Routes returns the HTTP handler for pacman requests. -// Mount this at /arch on your router. -func (h *ArchHandler) Routes() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet && r.Method != http.MethodHead { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - - path := strings.TrimPrefix(r.URL.Path, "/") - - // A decoded '?' or '#' would cut the upstream path short and cache one - // file under another file's name. - if containsPathTraversal(path) || strings.ContainsAny(path, "?#") { - http.Error(w, "invalid path", http.StatusBadRequest) - return - } - - repository, rest, ok := strings.Cut(path, "/") - upstreamURL, found := h.repositories[repository] - if !ok || rest == "" || !found { - http.NotFound(w, r) - return - } - - switch { - case isArchDatabase(rest): - h.handleDatabase(w, r, repository, upstreamURL, rest) - case isArchDatabaseSignature(rest): - h.handleDatabaseSignature(w, r, repository, upstreamURL, rest) - case isArchPackage(rest) || isArchPackageSignature(rest): - h.handlePackageDownload(w, r, repository, upstreamURL, rest) - default: - h.proxyFile(w, r, upstreamURL, rest) - } - }) -} - -// archDatabasePattern matches {repo}.db and {repo}.files, bare or as .tar.*. -var archDatabasePattern = regexp.MustCompile(`^[^/]+\.(?:db|files)(?:\.tar(?:\.(?:gz|xz|zst|bz2|lz4|lzo|lrz|lz|Z))?)?$`) - -func archBase(path string) string { - return path[strings.LastIndex(path, "/")+1:] -} - -func isArchDatabase(path string) bool { - return archDatabasePattern.MatchString(archBase(path)) -} - -func isArchDatabaseSignature(path string) bool { - base, ok := strings.CutSuffix(archBase(path), archSignatureSuffix) - return ok && archDatabasePattern.MatchString(base) -} - -// archPackagePattern matches {pkgname}-{pkgver}-{pkgrel}-{arch}.pkg.tar[.ext], -// e.g. lz4-1:1.10.0-2-x86_64.pkg.tar.zst. pkgname may contain '-', and pkgver -// may carry an epoch. -var archPackagePattern = regexp.MustCompile(`^(.+)-((?:\d+:)?[^-/:]+-[^-/:]+)-([^-/:]+)\.pkg\.tar(?:\.(?:zst|xz|gz|bz2|lz4|lzo|lrz|lz|Z))?$`) - -func isArchPackage(path string) bool { - return archPackagePattern.MatchString(archBase(path)) -} - -func isArchPackageSignature(path string) bool { - base, ok := strings.CutSuffix(archBase(path), archSignatureSuffix) - return ok && archPackagePattern.MatchString(base) -} - -// parseArchPackage returns name, pkgver-pkgrel and arch of a package or .sig. -func parseArchPackage(filename string) (name, version, arch string) { - matches := archPackagePattern.FindStringSubmatch(strings.TrimSuffix(filename, archSignatureSuffix)) - if len(matches) != archMatchCount { - return "", "", "" - } - return matches[1], matches[2], matches[3] -} - -// handlePackageDownload caches packages and their signatures. The cache key -// holds repository and full path, since two repositories may ship different -// bytes under one filename. -func (h *ArchHandler) handlePackageDownload(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - name, version, arch := parseArchPackage(archBase(path)) - if name == "" { - h.proxyFile(w, r, upstreamURL, path) - return - } - - downloadURL := upstreamURL + "/" + path - cacheFilename := repository + "/" + path - - h.proxy.Logger.Info("arch package download", - "repository", repository, "name", name, "version", version, "arch", arch, - "signature", strings.HasSuffix(path, archSignatureSuffix)) - - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), archEcosystem, name, version, cacheFilename, downloadURL) - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") - return - } - - if result.Artifact.MediaType == "" { - result.Artifact.MediaType = archPackageMediaType - } - ServeArtifactRequest(w, r, result) -} - -// handleMetadata serves databases unchanged through the metadata cache. -func (h *ArchHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, archEcosystem, - h.metadataCacheKey(repository, upstreamURL, path), archAccept) -} - -// archDatabase names a database and its signature in the cache. -type archDatabase struct { - dbKey, dbURL string - sigKey, sigURL string -} - -func (h *ArchHandler) database(repository, upstreamURL, path string) archDatabase { - return archDatabase{ - dbKey: h.metadataCacheKey(repository, upstreamURL, path), - dbURL: upstreamURL + "/" + path, - sigKey: h.metadataCacheKey(repository, upstreamURL, path+archSignatureSuffix), - sigURL: upstreamURL + "/" + path + archSignatureSuffix, - } -} - -func (d archDatabase) coalesceKey(kind string) string { - return metadataCoalesceKey(archEcosystem, d.dbKey, d.dbURL, archAccept, "identity", false) + "\x00" + kind -} - -// handleDatabase serves a database. Each refresh fetches the signature too, so -// a signature requested later still matches the database the client got. -func (h *ArchHandler) handleDatabase(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - p := h.proxy - if !p.CacheMetadata || p.DB == nil { - h.handleMetadata(w, r, repository, upstreamURL, path) - return - } - d := h.database(repository, upstreamURL, path) - if _, hit := p.cachedMetadataState(r.Context(), archEcosystem, d.dbKey, nil); hit != nil { - metrics.RecordMetadataCacheHit(archEcosystem, len(hit.body)) - h.writeMetadata(w, r, d.dbKey, *hit) - return - } - p.recordMetadataCacheMiss(archEcosystem) - res := h.sharedRefresh(r.Context(), d, true) - h.writeMetadata(w, r, d.dbKey, res) -} - -// handleDatabaseSignature serves the signature stored for the cached -// database's content, never one fetched for another version of it. -func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - p := h.proxy - if !p.CacheMetadata || p.DB == nil { - h.handleMetadata(w, r, repository, upstreamURL, path) - return - } - d := h.database(repository, upstreamURL, strings.TrimSuffix(path, archSignatureSuffix)) - if res, ok := h.cachedSignature(r.Context(), d); ok { - metrics.RecordMetadataCacheHit(archEcosystem, len(res.body)) - h.writeMetadata(w, r, d.sigKey, res) - return - } - p.recordMetadataCacheMiss(archEcosystem) - // Nothing stored for the cached database, e.g. its signature fetch failed - // during the last refresh. The client may already hold that database, so - // only a signature for it will do; if the refresh replaces it, fail. - served := h.databaseDigest(d) - res, _ := p.coalesceMetadata(r.Context(), d.coalesceKey("signature\x00"+served), func(ctx context.Context) metadataResult { - if res, ok := h.storedSignature(ctx, d, served); ok { - return res - } - if res := h.sharedRefresh(ctx, d, false); res.err != nil { - return res - } - if served == "" { - if res, ok := h.cachedSignature(ctx, d); ok { - return res - } - } else if res, ok := h.storedSignature(ctx, d, served); ok { - return res - } - return metadataResult{err: errArchSignatureUnpaired} - }) - h.writeMetadata(w, r, d.sigKey, res) -} - -var errArchSignatureUnpaired = errors.New("database signature could not be paired with the database") - -// sharedRefresh runs one refresh per database at a time, so a slow fetch -// cannot store an older database over one cached after it started. With -// ifStale, a refresh that finds the database fresh again skips the upstream. -func (h *ArchHandler) sharedRefresh(ctx context.Context, d archDatabase, ifStale bool) metadataResult { - p := h.proxy - res, _ := p.coalesceMetadata(ctx, d.coalesceKey("database"), func(ctx context.Context) metadataResult { - entry, hit := p.cachedMetadataState(ctx, archEcosystem, d.dbKey, nil) - if hit != nil && ifStale { - return *hit - } - res, _ := h.refresh(ctx, d, entry) - return res - }) - return res -} - -// archPairAttempts bounds refetches while the mirror replaces the database. -const archPairAttempts = 3 - -// refresh fetches the database, then its signature, and stores the signature -// for the database's content. It returns the database as cached and why no -// signature could be stored, if so. -func (h *ArchHandler) refresh(ctx context.Context, d archDatabase, entry *database.MetadataCacheEntry) (metadataResult, error) { - p := h.proxy - res, row := h.fetchDatabase(ctx, d, entry) - if res.err != nil { - return res, res.err - } - for range archPairAttempts { - if row == nil || !row.ContentDigest.Valid { - return res, errArchSignatureUnpaired - } - dbDigest := row.ContentDigest.String - if _, ok := h.storedSignature(ctx, d, dbDigest); ok { - return res, nil - } - - sig, err := p.fetchUpstreamMetadata(ctx, d.sigURL, nil, archAccept, "identity") - if errors.Is(err, ErrUpstreamNotFound) { - h.storeSignature(ctx, d, dbDigest, nil) - return res, err - } - if err != nil { - return res, err - } - - // The mirror may have replaced the database in the meantime. - again, err := p.fetchUpstreamMetadata(ctx, d.dbURL, row, archAccept, "identity") - if err != nil { - return res, err - } - if archDigest(again.body) == dbDigest { - h.storeSignature(ctx, d, dbDigest, sig) - return res, nil - } - if res, row = h.storeDatabase(ctx, d, again); res.err != nil { - return res, res.err - } - } - return res, errArchSignatureUnpaired -} - -// fetchDatabase fetches and caches the database, falling back to the cached -// copy when the upstream fails. -func (h *ArchHandler) fetchDatabase(ctx context.Context, d archDatabase, entry *database.MetadataCacheEntry) (metadataResult, *database.MetadataCacheEntry) { - p := h.proxy - meta, err := p.fetchUpstreamMetadata(ctx, d.dbURL, entry, archAccept, "identity") - if errors.Is(err, errStale304) { - meta, err = p.fetchUpstreamMetadata(ctx, d.dbURL, nil, archAccept, "identity") - } - if err == nil { - return h.storeDatabase(ctx, d, meta) - } - if entry == nil { - return metadataResult{err: fmt.Errorf("upstream failed and no cached metadata: %w", err)}, nil - } - p.Logger.Warn("upstream metadata fetch failed, checking cache", "ecosystem", archEcosystem, "key", d.dbKey, "error", err) - return h.readDatabase(ctx, p.currentMetadataEntry(archEcosystem, d.dbKey, entry), err) -} - -// storeDatabase caches meta under its digest and returns the database the -// cache row now describes. That is meta unless the write failed; bytes the -// row does not describe would be paired with the wrong signature. -func (h *ArchHandler) storeDatabase(ctx context.Context, d archDatabase, meta *upstreamMetadata) (metadataResult, *database.MetadataCacheEntry) { - p := h.proxy - path := archBlobPath(d.dbKey, archDigest(meta.body)) - defer h.lock(d.dbKey)() - - previous, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) - if previous != nil && previous.StoragePath == path && h.blobExists(ctx, path) { - // Same bytes: a failed rewrite would delete the blob the row uses. - h.touchDatabase(previous, meta) - } else { - p.cacheMetadataBlob(ctx, archEcosystem, d.dbKey, path, meta) - } - row, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) - if row == nil { - // Nothing cached at all: serve it like an uncached proxy would. - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding}, nil - } - if row.StoragePath != path { - return h.readDatabase(ctx, row, errArchDatabaseNotCached) - } - if previous != nil && previous.StoragePath != path { - _ = p.Storage.Delete(ctx, previous.StoragePath) - } - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding}, row -} - -// blobExists reports whether path is stored. Size eviction can remove a blob -// while a refresh re-creates its row, and only a rewrite brings it back. -func (h *ArchHandler) blobExists(ctx context.Context, path string) bool { - ok, err := h.proxy.Storage.Exists(ctx, path) - return ok || err != nil -} - -// touchDatabase marks an unchanged cached database as fetched now. -func (h *ArchHandler) touchDatabase(row *database.MetadataCacheEntry, meta *upstreamMetadata) { - updated := *row - updated.ETag = sql.NullString{String: meta.etag, Valid: meta.etag != ""} - updated.LastModified = sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()} - updated.FetchedAt = sql.NullTime{Time: time.Now(), Valid: true} - if err := h.proxy.DB.UpsertMetadataCache(&updated); err != nil { - h.proxy.Logger.Warn("failed to record cached metadata", "ecosystem", archEcosystem, "key", row.Name, "error", err) - } -} - -var errArchDatabaseNotCached = errors.New("database could not be cached") - -// readDatabase returns the cached database row's bytes, or cause if they are -// unreadable. -func (h *ArchHandler) readDatabase(ctx context.Context, row *database.MetadataCacheEntry, cause error) (metadataResult, *database.MetadataCacheEntry) { - body, contentType, err := h.proxy.readCachedMetadata(ctx, row, nil) - if err != nil { - return metadataResult{err: fmt.Errorf("cached database unusable (%v): %w", err, cause)}, nil - } - return metadataResult{body: body, contentType: contentType, contentEncoding: row.ContentEncoding.String}, row -} - -// lock serializes cache writes for one database and its signature, so a row -// never points at a blob another write has just removed. -func (h *ArchHandler) lock(dbKey string) func() { - m, _ := h.locks.LoadOrStore(dbKey, &sync.Mutex{}) - mu := m.(*sync.Mutex) - mu.Lock() - return mu.Unlock -} - -// cachedSignature returns the signature stored for the cached database. -func (h *ArchHandler) cachedSignature(ctx context.Context, d archDatabase) (metadataResult, bool) { - dbDigest := h.databaseDigest(d) - if dbDigest == "" { - return metadataResult{}, false - } - return h.storedSignature(ctx, d, dbDigest) -} - -// databaseDigest returns the cached database's content digest, or "". -func (h *ArchHandler) databaseDigest(d archDatabase) string { - current, _ := h.proxy.DB.GetMetadataCache(archEcosystem, d.dbKey) - if current == nil || !current.ContentDigest.Valid { - return "" - } - return current.ContentDigest.String -} - -// storedSignature returns the signature stored for dbDigest. The storage -// path names the digest, so a refresh with unchanged bytes keeps it. -func (h *ArchHandler) storedSignature(ctx context.Context, d archDatabase, dbDigest string) (metadataResult, bool) { - p := h.proxy - entry, _ := p.DB.GetMetadataCache(archEcosystem, d.sigKey) - if entry == nil { - return metadataResult{}, false - } - switch entry.StoragePath { - case archMissingSignaturePath(d, dbDigest): - // Ask again after the TTL, in case the 404 was transient. - if p.MetadataTTL <= 0 || !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { - return metadataResult{}, false - } - return metadataResult{err: ErrUpstreamNotFound}, true - case archSignaturePath(d, dbDigest): - body, contentType, err := p.readCachedMetadata(ctx, entry, nil) - if err != nil { - return metadataResult{}, false - } - return metadataResult{body: body, contentType: contentType, contentEncoding: entry.ContentEncoding.String}, true - } - return metadataResult{}, false -} - -// storeSignature stores sig for dbDigest, or that it is missing when sig is -// nil, and drops the one stored for an earlier database. -func (h *ArchHandler) storeSignature(ctx context.Context, d archDatabase, dbDigest string, sig *upstreamMetadata) { - p := h.proxy - path := archSignaturePath(d, dbDigest) - if sig == nil { - path = archMissingSignaturePath(d, dbDigest) - sig = &upstreamMetadata{contentType: archPackageMediaType} - } - defer h.lock(d.dbKey)() - - previous, _ := p.DB.GetMetadataCache(archEcosystem, d.sigKey) - p.cacheMetadataBlob(ctx, archEcosystem, d.sigKey, path, sig) - if previous == nil || previous.StoragePath == path { - return - } - if current, _ := p.DB.GetMetadataCache(archEcosystem, d.sigKey); current != nil && current.StoragePath == path { - _ = p.Storage.Delete(ctx, previous.StoragePath) - } -} - -// archBlobPath names a database or signature blob after the database digest -// it belongs to, so a cache row and its bytes always agree. -func archBlobPath(key, dbDigest string) string { - return "_metadata/" + archEcosystem + "/" + key + "/" + strings.TrimPrefix(dbDigest, "sha256:") -} - -func archSignaturePath(d archDatabase, dbDigest string) string { - return archBlobPath(d.sigKey, dbDigest) -} - -func archMissingSignaturePath(d archDatabase, dbDigest string) string { - return archSignaturePath(d, dbDigest) + "-missing" -} - -func archDigest(body []byte) string { - digest := sha256.Sum256(body) - return "sha256:" + hex.EncodeToString(digest[:]) -} - -func (h *ArchHandler) writeMetadata(w http.ResponseWriter, r *http.Request, cacheKey string, res metadataResult) { - if res.err != nil { - if errors.Is(res.err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if writeUpstreamHeld(w, res.err) { - return - } - h.proxy.Logger.Error("metadata fetch failed", "error", res.err) - http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) - return - } - h.proxy.writeMetadataCachedResponseWithEncoding(w, r, archEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) -} - -// metadataCacheKey hashes repository, upstream URL and path, as APKHandler does. -func (h *ArchHandler) metadataCacheKey(repository, upstreamURL, path string) string { - identity := repository + "\x00" + upstreamURL + "\x00" + path - digest := sha256.Sum256([]byte(identity)) - return hex.EncodeToString(digest[:]) -} - -// proxyFile passes other files (lastupdate, ISOs) through uncached. -func (h *ArchHandler) proxyFile(w http.ResponseWriter, r *http.Request, upstreamURL, path string) { - h.proxy.ProxyFile(w, r, upstreamURL+"/"+path) -} diff --git a/internal/handler/arch_concurrency_test.go b/internal/handler/arch_concurrency_test.go deleted file mode 100644 index a2f374e..0000000 --- a/internal/handler/arch_concurrency_test.go +++ /dev/null @@ -1,315 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "net/http" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/storage" -) - -// archHookStorage wraps a storage with hooks to force interleavings. -type archHookStorage struct { - storage.Storage - failStore func(path string) error - afterStore func(path string) - beforeDelete func(path string) -} - -func (s *archHookStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { - if s.failStore != nil { - if err := s.failStore(path); err != nil { - return 0, "", err - } - } - n, hash, err := s.Storage.Store(ctx, path, r) - if s.afterStore != nil { - s.afterStore(path) - } - return n, hash, err -} - -func (s *archHookStorage) Delete(ctx context.Context, path string) error { - if s.beforeDelete != nil { - s.beforeDelete(path) - } - return s.Storage.Delete(ctx, path) -} - -const archTestDB, archTestSig = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" - -// pauseOnce returns a hook that stops the first call for a path starting with -// prefix until release -// closes or a short timeout passes, so a test does not hang when locking -// keeps the other writer out. -func pauseOnce(prefix string, paused, release chan struct{}) func(string) { - var fired atomic.Bool - return func(p string) { - if !strings.HasPrefix(p, prefix) || !fired.CompareAndSwap(false, true) { - return - } - close(paused) - select { - case <-release: - case <-time.After(200 * time.Millisecond): - } - } -} - -// waitPaused waits until a pauseOnce hook fired. -func waitPaused(t *testing.T, paused <-chan struct{}) { - t.Helper() - select { - case <-paused: - case <-time.After(5 * time.Second): - t.Fatal("hook never fired") - } -} - -// waitBriefly waits for done, or gives up after a moment when a lock holds the -// call back. -func waitBriefly(done <-chan struct{}) { - select { - case <-done: - case <-time.After(300 * time.Millisecond): - } -} - -// TestArchHandler_ReplicasWritingOutOfOrder covers two proxy replicas sharing -// database and storage, whose database writes finish out of order: the cache -// row must never describe other bytes than its blob, or every client gets the -// newest database with an older signature. -func TestArchHandler_ReplicasWritingOutOfOrder(t *testing.T) { - upstream := newArchUpstream(t, map[string][]byte{archTestDB: []byte("db v1"), archTestSig: []byte("sig v1")}) - upstream.broken[archTestSig] = true - proxy := newArchTestProxy(t, upstream) - replicaA := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - replicaB := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - d := replicaA.database("custom", upstream.URL, archTestDB[1:]) - - assertArchBody(t, serveArchRequest(replicaA, http.MethodGet, "/custom"+archTestDB, nil), "db v1", []byte("db v1")) - upstream.mu.Lock() - upstream.files[archTestDB], upstream.files[archTestSig] = []byte("db v2"), []byte("sig v2") - upstream.broken[archTestSig] = false - upstream.mu.Unlock() - ageMetadata(t, proxy, replicaA, "custom", upstream.URL, archTestDB[1:]) - - // Replica A's signature recovery stores db v2 and stalls before its row - // update. - paused, release := make(chan struct{}), make(chan struct{}) - proxy.Storage = &archHookStorage{Storage: proxy.Storage, - afterStore: pauseOnce("_metadata/"+archEcosystem+"/"+d.dbKey+"/", paused, release)} - recovered := make(chan struct{}) - go func() { - serveArchRequest(replicaA, http.MethodGet, "/custom"+archTestSig, nil) - close(recovered) - }() - waitPaused(t, paused) - - // Replica B pairs v2, then caches v3 whose signature fetch fails. - assertArchBody(t, serveArchRequest(replicaB, http.MethodGet, "/custom"+archTestDB, nil), "B db v2", []byte("db v2")) - assertArchBody(t, serveArchRequest(replicaB, http.MethodGet, "/custom"+archTestSig, nil), "B sig v2", []byte("sig v2")) - upstream.mu.Lock() - upstream.files[archTestDB], upstream.files[archTestSig] = []byte("db v3"), []byte("sig v3") - upstream.broken[archTestSig] = true - upstream.mu.Unlock() - ageMetadata(t, proxy, replicaB, "custom", upstream.URL, archTestDB[1:]) - assertArchBody(t, serveArchRequest(replicaB, http.MethodGet, "/custom"+archTestDB, nil), "B db v3", []byte("db v3")) - - close(release) - <-recovered - - db := serveArchRequest(replicaB, http.MethodGet, "/custom"+archTestDB, nil) - sig := serveArchRequest(replicaB, http.MethodGet, "/custom"+archTestSig, nil) - if sig.Code == http.StatusOK && strings.Replace(db.Body.String(), "db", "sig", 1) != sig.Body.String() { - t.Errorf("%q served with %q", db.Body.String(), sig.Body.String()) - } -} - -// TestArchHandler_FailedDatabaseWriteKeepsPair covers a database refresh whose -// cache write fails: the proxy must not hand out the new database while the -// cache still pairs the old one with its signature. -func TestArchHandler_FailedDatabaseWriteKeepsPair(t *testing.T) { - upstream := newArchUpstream(t, map[string][]byte{archTestDB: []byte("db v1"), archTestSig: []byte("sig v1")}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - d := h.database("custom", upstream.URL, archTestDB[1:]) - var fail atomic.Bool - proxy.Storage = &archHookStorage{Storage: proxy.Storage, failStore: func(path string) error { - if fail.Load() && strings.HasPrefix(path, archBlobPath(d.dbKey, "")) { - return errors.New("disk full") - } - return nil - }} - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil), "db v1", []byte("db v1")) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil), "sig v1", []byte("sig v1")) - upstream.set(archTestDB, []byte("db v2")) - upstream.set(archTestSig, []byte("sig v2")) - ageMetadata(t, proxy, h, "custom", upstream.URL, archTestDB[1:]) - fail.Store(true) - - db := serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil) - sig := serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil) - if db.Code == http.StatusOK && sig.Code == http.StatusOK && strings.Replace(db.Body.String(), "db", "sig", 1) != sig.Body.String() { - t.Errorf("%q served with %q", db.Body.String(), sig.Body.String()) - } -} - -// TestArchHandler_ConcurrentSignatureWritesLeaveOneBlob covers two signature -// writes overlapping: afterwards exactly one signature blob is left and the -// cache row points at it. -func TestArchHandler_ConcurrentSignatureWritesLeaveOneBlob(t *testing.T) { - d1, d2, d3 := archDigest([]byte("1")), archDigest([]byte("2")), archDigest([]byte("3")) - - for _, tt := range []struct { - name string - hook func(h *ArchHandler, d archDatabase, s *archHookStorage, paused, release chan struct{}) - first string // digest stored by the stalled write - other string // digest stored by the write overtaking it - }{ - {"stalled after store", func(_ *ArchHandler, d archDatabase, s *archHookStorage, paused, release chan struct{}) { - s.afterStore = pauseOnce(archSignaturePath(d, d2), paused, release) - }, d2, d3}, - {"stalled before delete", func(_ *ArchHandler, d archDatabase, s *archHookStorage, paused, release chan struct{}) { - s.beforeDelete = pauseOnce(archSignaturePath(d, d1), paused, release) - }, d2, d1}, - } { - t.Run(tt.name, func(t *testing.T) { - upstream := newArchUpstream(t, map[string][]byte{}) - proxy := newArchTestProxy(t, upstream) - mock := proxy.Storage.(*mockStorage) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - d := h.database("custom", upstream.URL, archTestDB[1:]) - ctx := context.Background() - h.storeSignature(ctx, d, d1, &upstreamMetadata{body: []byte("sig 1")}) - - hooks := &archHookStorage{Storage: mock} - proxy.Storage = hooks - paused, release := make(chan struct{}), make(chan struct{}) - tt.hook(h, d, hooks, paused, release) - - stalled := make(chan struct{}) - go func() { - h.storeSignature(ctx, d, tt.first, &upstreamMetadata{body: []byte("sig first")}) - close(stalled) - }() - waitPaused(t, paused) - overtaking := make(chan struct{}) - go func() { - h.storeSignature(ctx, d, tt.other, &upstreamMetadata{body: []byte("sig other")}) - close(overtaking) - }() - waitBriefly(overtaking) - close(release) - <-stalled - <-overtaking - - row, _ := proxy.DB.GetMetadataCache(archEcosystem, d.sigKey) - if _, err := mock.Open(ctx, row.StoragePath); err != nil { - t.Errorf("row points at %s, which is gone: %v", row.StoragePath, err) - } - mock.mu.Lock() - defer mock.mu.Unlock() - var blobs []string - for path := range mock.files { - if strings.HasPrefix(path, archBlobPath(d.sigKey, "")) { - blobs = append(blobs, path) - } - } - if len(blobs) != 1 { - t.Errorf("signature blobs = %v, want only the row's", blobs) - } - }) - } -} - -// TestArchHandler_MissingSignatureRecheckedAfterTTL covers a 404 for the -// signature that was only transient: once the TTL is over the proxy asks -// again instead of answering 404 until the next database is published. -func TestArchHandler_MissingSignatureRecheckedAfterTTL(t *testing.T) { - upstream := newArchUpstream(t, map[string][]byte{archTestDB: []byte("db v1")}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil), "db", []byte("db v1")) - if w := serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil); w.Code != http.StatusNotFound { - t.Fatalf("signature: status = %d, want 404", w.Code) - } - - upstream.set(archTestSig, []byte("sig v1")) - ageMetadata(t, proxy, h, "custom", upstream.URL, archTestDB[1:]) - ageMetadata(t, proxy, h, "custom", upstream.URL, archTestSig[1:]) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil), "db refreshed", []byte("db v1")) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil), "signature after TTL", []byte("sig v1")) -} - -// TestArchHandler_SlowFetchDoesNotOverwriteNewerDatabase covers a signature -// recovery whose database fetch is still in flight while other clients -// refresh to newer databases. Its result must not replace the newer cached -// database, or a client holding that one gets an older signature. -func TestArchHandler_SlowFetchDoesNotOverwriteNewerDatabase(t *testing.T) { - upstream := newArchUpstream(t, map[string][]byte{archTestDB: []byte("db v1"), archTestSig: []byte("sig v1")}) - upstream.broken[archTestSig] = true - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil), "db v1", []byte("db v1")) - upstream.mu.Lock() - upstream.files[archTestDB], upstream.files[archTestSig] = []byte("db v2"), []byte("sig v2") - upstream.broken[archTestSig] = false - upstream.mu.Unlock() - ageMetadata(t, proxy, h, "custom", upstream.URL, archTestDB[1:]) - - // Client A's signature recovery has fetched db v2; the response is slow. - paused, release := make(chan struct{}), make(chan struct{}) - upstream.mu.Lock() - upstream.served = pauseOnce(archTestDB, paused, release) - upstream.mu.Unlock() - recovered := make(chan struct{}) - go func() { - serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil) - close(recovered) - }() - waitPaused(t, paused) - - // Client B refreshes meanwhile, then the upstream moves to v3, whose - // signature fails, and client C refreshes to it. - b := make(chan struct{}) - go func() { - serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil) - serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil) - close(b) - }() - waitBriefly(b) - upstream.mu.Lock() - upstream.files[archTestDB], upstream.files[archTestSig] = []byte("db v3"), []byte("sig v3") - upstream.broken[archTestSig] = true - upstream.mu.Unlock() - ageMetadata(t, proxy, h, "custom", upstream.URL, archTestDB[1:]) - var cDB string - c := make(chan struct{}) - go func() { - cDB = serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil).Body.String() - close(c) - }() - waitBriefly(c) - - close(release) - <-recovered - <-b - <-c - - if sig := serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil); sig.Code == http.StatusOK && strings.Replace(cDB, "db", "sig", 1) != sig.Body.String() { - t.Errorf("client holding %q got %q", cDB, sig.Body.String()) - } - if got := serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil).Body.String(); got < cDB { - t.Errorf("cache went back to %q after serving %q", got, cDB) - } -} diff --git a/internal/handler/arch_test.go b/internal/handler/arch_test.go deleted file mode 100644 index 3affad5..0000000 --- a/internal/handler/arch_test.go +++ /dev/null @@ -1,650 +0,0 @@ -package handler - -import ( - "fmt" - "net/http" - "net/http/httptest" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/registries/fetch" -) - -func TestParseArchPackage(t *testing.T) { - assertPathParser(t, "parseArchPackage", parseArchPackage, []pathParseCase{ - {"bash-5.2.037-1-x86_64.pkg.tar.zst", "bash", "5.2.037-1", "x86_64"}, - {"python-pip-24.3.1-1-any.pkg.tar.zst", "python-pip", "24.3.1-1", "any"}, - {"lz4-1:1.10.0-2-x86_64.pkg.tar.zst", "lz4", "1:1.10.0-2", "x86_64"}, - {"wpa_supplicant-2:2.12-1-x86_64.pkg.tar.zst", "wpa_supplicant", "2:2.12-1", "x86_64"}, - {"linux-aarch64-6.12.1-1-aarch64.pkg.tar.xz", "linux-aarch64", "6.12.1-1", "aarch64"}, - {"libsigc++-3.6.0-1.1-x86_64.pkg.tar.lz4", "libsigc++", "3.6.0-1.1", "x86_64"}, - {"plain-1.0-1-x86_64.pkg.tar", "plain", "1.0-1", "x86_64"}, - {"bash-5.2.037-1-x86_64.pkg.tar.zst.sig", "bash", "5.2.037-1", "x86_64"}, - {"bash-x86_64.pkg.tar.zst", "", "", ""}, - {"bash-5.2.037-1-x86_64.tar.zst", "", "", ""}, - {"core.db", "", "", ""}, - }) -} - -func TestArchFileClassification(t *testing.T) { - tests := []struct { - path string - database, databaseSig, pkg, pkgSig bool - }{ - {path: "core/os/x86_64/core.db", database: true}, - {path: "core/os/x86_64/core.db.tar.gz", database: true}, - {path: "core/os/x86_64/core.files", database: true}, - {path: "core/os/x86_64/core.files.tar.zst", database: true}, - {path: "aarch64/core/core.db", database: true}, - {path: "core/os/x86_64/core.db.sig", databaseSig: true}, - {path: "custom/os/x86_64/custom.db.tar.xz.sig", databaseSig: true}, - {path: "core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst", pkg: true}, - {path: "core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst.sig", pkgSig: true}, - {path: "core/os/x86_64/core.db.tar.gz.old"}, - {path: "lastupdate"}, - {path: "iso/latest/archlinux-x86_64.iso"}, - } - for _, tt := range tests { - if got := isArchDatabase(tt.path); got != tt.database { - t.Errorf("isArchDatabase(%q) = %v, want %v", tt.path, got, tt.database) - } - if got := isArchDatabaseSignature(tt.path); got != tt.databaseSig { - t.Errorf("isArchDatabaseSignature(%q) = %v, want %v", tt.path, got, tt.databaseSig) - } - if got := isArchPackage(tt.path); got != tt.pkg { - t.Errorf("isArchPackage(%q) = %v, want %v", tt.path, got, tt.pkg) - } - if got := isArchPackageSignature(tt.path); got != tt.pkgSig { - t.Errorf("isArchPackageSignature(%q) = %v, want %v", tt.path, got, tt.pkgSig) - } - } -} - -// TestArchPURL pins the package URL type: git-pkgs/purl maps the "arch" -// ecosystem to pkg:alpm with the "arch" namespace. -func TestArchPURL(t *testing.T) { - pkg, version := packageurl.MakeCacheStrings(archEcosystem, "lz4", "1:1.10.0-2") - if pkg != "pkg:alpm/arch/lz4" { - t.Errorf("package PURL = %q, want %q", pkg, "pkg:alpm/arch/lz4") - } - if version != "pkg:alpm/arch/lz4@1:1.10.0-2" { - t.Errorf("version PURL = %q, want %q", version, "pkg:alpm/arch/lz4@1:1.10.0-2") - } -} - -func TestArchHandler_Routes(t *testing.T) { - h := NewArchHandler(nil, "http://localhost:8080", nil) - assertRoutesBasics(t, h.Routes(), "/archlinux/core/os/x86_64/core.db", "/archlinux/core/../../../etc/passwd") -} - -func TestArchHandler_DefaultsToGeoMirror(t *testing.T) { - h := NewArchHandler(nil, "http://localhost:8080", nil) - if got := h.repositories[defaultArchRepositoryName]; got != defaultArchUpstream { - t.Errorf("default repository = %q, want %q", got, defaultArchUpstream) - } - if len(h.repositories) != 1 { - t.Errorf("repositories = %v, want only the default", h.repositories) - } -} - -func TestArchHandler_UnknownRepositoryReturns404(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - h := NewArchHandler(proxy, "http://localhost:8080", map[string]string{"archlinux": "https://example.test"}) - - for _, target := range []string{"/unknown/core/os/x86_64/core.db", "/archlinux", "/"} { - if w := serveArchRequest(h, http.MethodGet, target, nil); w.Code != http.StatusNotFound { - t.Errorf("%s: status = %d, want 404", target, w.Code) - } - } -} - -// archUpstream is a fake pacman mirror. Files can be swapped, single paths -// broken and the mirror taken offline; requests are counted per path. -type archUpstream struct { - *httptest.Server - mu sync.Mutex - files map[string][]byte - modified map[string]time.Time - hits map[string]int - broken map[string]bool - served func(path string) // called after a file is served, without mu - available atomic.Bool - auth atomic.Value -} - -func newArchUpstream(t *testing.T, files map[string][]byte) *archUpstream { - t.Helper() - u := &archUpstream{files: files, modified: map[string]time.Time{}, hits: map[string]int{}, broken: map[string]bool{}} - u.available.Store(true) - u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !u.available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - u.auth.Store(r.Header.Get("Authorization")) - u.mu.Lock() - data, ok := u.files[r.URL.Path] - modified := u.modified[r.URL.Path] - broken := u.broken[r.URL.Path] - served := u.served - u.hits[r.URL.Path]++ - u.mu.Unlock() - if broken { - http.Error(w, "broken", http.StatusInternalServerError) - return - } - if !ok { - http.NotFound(w, r) - return - } - if !modified.IsZero() { - w.Header().Set("Last-Modified", modified.UTC().Format(http.TimeFormat)) - } - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = w.Write(data) - if served != nil { - served(r.URL.Path) - } - })) - t.Cleanup(u.Close) - return u -} - -func (u *archUpstream) set(path string, data []byte) { - u.mu.Lock() - defer u.mu.Unlock() - u.files[path] = data -} - -func (u *archUpstream) hitCount(path string) int { - u.mu.Lock() - defer u.mu.Unlock() - return u.hits[path] -} - -func newArchTestProxy(t *testing.T, upstream *archUpstream) *Proxy { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - return proxy -} - -func assertArchBody(t *testing.T, w *httptest.ResponseRecorder, label string, want []byte) { - t.Helper() - if w.Code != http.StatusOK { - t.Fatalf("%s: status = %d, want 200: %s", label, w.Code, w.Body.String()) - } - if got := w.Body.String(); got != string(want) { - t.Errorf("%s: body altered:\ngot %q\nwant %q", label, got, want) - } -} - -// TestArchHandler_DatabasesServedUnchanged covers database and file-list -// downloads under the names pacman uses (core.db, core.files) and their -// .tar.* forms: bytes are served unchanged so signatures stay valid, cached -// copies are served within the TTL without contacting the upstream, and the -// cache keeps serving them when the upstream is down. -func TestArchHandler_DatabasesServedUnchanged(t *testing.T) { - files := map[string][]byte{ - "/core/os/x86_64/core.db": []byte("\x1f\x8b\x08core-db-bytes"), - "/core/os/x86_64/core.db.tar.gz": []byte("\x1f\x8b\x08core-db-bytes"), - "/core/os/x86_64/core.db.sig": []byte("core-db-signature"), - "/core/os/x86_64/core.files": []byte("\x1f\x8b\x08core-files-bytes"), - "/core/os/x86_64/core.files.tar.zst": []byte("\x28\xb5\x2f\xfdcore-files-zst"), - } - upstream := newArchUpstream(t, files) - proxy := newArchTestProxy(t, upstream) - authorizeUpstream(proxy, "Bearer arch-token") - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) - - // The database goes first, as pacman requests it. - paths := []string{ - "/core/os/x86_64/core.db", "/core/os/x86_64/core.db.sig", "/core/os/x86_64/core.db.tar.gz", - "/core/os/x86_64/core.files", "/core/os/x86_64/core.files.tar.zst", - } - for _, path := range paths { - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), path, files[path]) - } - if got, _ := upstream.auth.Load().(string); got != "Bearer arch-token" { - t.Errorf("Authorization = %q, want %q", got, "Bearer arch-token") - } - - // core.db is fetched twice on its first request: once more after its - // signature, to make sure the two belong together. - wantHits := map[string]int{"/core/os/x86_64/core.db": 2} - for _, path := range paths { - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), path+" cached", files[path]) - want := max(wantHits[path], 1) - if got := upstream.hitCount(path); got != want { - t.Errorf("%s: upstream hits = %d, want %d (second read from cache)", path, got, want) - } - } - - upstream.available.Store(false) - for _, path := range paths { - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), path+" offline", files[path]) - } -} - -// TestArchHandler_StaleDatabaseServedWhenUpstreamDown covers the stale -// fallback after the TTL. -func TestArchHandler_StaleDatabaseServedWhenUpstreamDown(t *testing.T) { - db := []byte("core-db-bytes") - upstream := newArchUpstream(t, map[string][]byte{"/core/os/x86_64/core.db": db}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil), "seed", db) - ageMetadata(t, proxy, h, "archlinux", upstream.URL, "core/os/x86_64/core.db") - upstream.available.Store(false) - - w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil) - assertArchBody(t, w, "stale", db) - if got := w.Header().Get("Warning"); got == "" { - t.Error("stale database served without a Warning header") - } -} - -// TestArchHandler_DatabaseIfModifiedSince covers pacman's database refresh, -// which sends If-Modified-Since with the local copy's time. -func TestArchHandler_DatabaseIfModifiedSince(t *testing.T) { - modified := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) - upstream := newArchUpstream(t, map[string][]byte{"/core/os/x86_64/core.db": []byte("core-db-bytes")}) - upstream.modified["/core/os/x86_64/core.db"] = modified - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) - - target := "/archlinux/core/os/x86_64/core.db" - assertArchBody(t, serveArchRequest(h, http.MethodGet, target, nil), "seed", []byte("core-db-bytes")) - - upToDate := http.Header{"If-Modified-Since": {modified.Format(http.TimeFormat)}} - if w := serveArchRequest(h, http.MethodGet, target, upToDate); w.Code != http.StatusNotModified { - t.Errorf("unchanged database: status = %d, want 304", w.Code) - } - older := http.Header{"If-Modified-Since": {modified.Add(-time.Hour).Format(http.TimeFormat)}} - assertArchBody(t, serveArchRequest(h, http.MethodGet, target, older), "older local copy", []byte("core-db-bytes")) -} - -// TestArchHandler_MissingDatabaseSignature covers the official repositories, -// which sign no databases: pacman with DatabaseOptional asks for core.db.sig -// and must get a 404. The database refresh found it missing, so the answer -// needs no upstream and holds offline too. -func TestArchHandler_MissingDatabaseSignature(t *testing.T) { - upstream := newArchUpstream(t, map[string][]byte{"/core/os/x86_64/core.db": []byte("core-db-bytes")}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil), "db", []byte("core-db-bytes")) - if w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db.sig", nil); w.Code != http.StatusNotFound { - t.Errorf("missing signature: status = %d, want 404", w.Code) - } - if got := upstream.hitCount("/core/os/x86_64/core.db.sig"); got != 1 { - t.Errorf("signature upstream hits = %d, want 1 (asked once, with the database)", got) - } - - upstream.available.Store(false) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil), "db offline", []byte("core-db-bytes")) - if w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db.sig", nil); w.Code != http.StatusNotFound { - t.Errorf("missing signature offline: status = %d, want 404", w.Code) - } -} - -// TestArchHandler_DatabaseSignatureFollowsDatabase covers custom repositories -// with signed databases. The database and its signature are separate cache -// entries; when the database is refreshed, the signature must be refreshed -// with it even inside its own TTL, or pacman would get a new database with -// the old signature. While the database is not refreshed, the cached -// signature keeps matching it and is served without asking the upstream. -func TestArchHandler_DatabaseSignatureFollowsDatabase(t *testing.T) { - const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" - upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v1", []byte("db v1")) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v1", []byte("sig v1")) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v1 again", []byte("sig v1")) - if got := upstream.hitCount(sigPath); got != 1 { - t.Errorf("signature upstream hits = %d, want 1 while the database is unchanged", got) - } - - // Upstream publishes a new database. The cached database expires before - // the signature does. - upstream.set(dbPath, []byte("db v2")) - upstream.set(sigPath, []byte("sig v2")) - ageMetadata(t, proxy, h, "custom", upstream.URL, dbPath[1:]) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v2", []byte("db v2")) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig after db refresh", []byte("sig v2")) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v2 again", []byte("sig v2")) - if got := upstream.hitCount(sigPath); got != 2 { - t.Errorf("signature upstream hits = %d, want 2", got) - } -} - -// TestArchHandler_OutdatedSignatureNeverServed covers a signature fetch that -// fails while the database is refreshed. The cached signature belongs to the -// old database, and pacman rejects a present but invalid signature even with -// DatabaseOptional, so the proxy must answer 404 or 502 rather than fall back -// to it. -func TestArchHandler_OutdatedSignatureNeverServed(t *testing.T) { - const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" - - for _, tt := range []struct { - name string - breakSig func(u *archUpstream) - want int - }{ - {"signature removed upstream", func(u *archUpstream) { delete(u.files, sigPath) }, http.StatusNotFound}, - {"signature fetch fails", func(u *archUpstream) { u.broken[sigPath] = true }, http.StatusBadGateway}, - } { - t.Run(tt.name, func(t *testing.T) { - upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v1", []byte("db v1")) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v1", []byte("sig v1")) - - upstream.mu.Lock() - upstream.files[dbPath] = []byte("db v2") - tt.breakSig(upstream) - upstream.mu.Unlock() - ageMetadata(t, proxy, h, "custom", upstream.URL, dbPath[1:]) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v2", []byte("db v2")) - - for i := range 2 { - w := serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil) - if w.Code != tt.want { - t.Errorf("request %d: status = %d, body = %q, want %d", i, w.Code, w.Body.String(), tt.want) - } - } - }) - } -} - -// TestArchHandler_DelayedSignatureMatchesCachedDatabase covers a signature -// first requested after the upstream moved on: it must be the one for the -// database the proxy still serves, not the upstream's new one. -func TestArchHandler_DelayedSignatureMatchesCachedDatabase(t *testing.T) { - const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" - upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v1", []byte("db v1")) - - upstream.set(dbPath, []byte("db v2")) - upstream.set(sigPath, []byte("sig v2")) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db within TTL", []byte("db v1")) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "first signature request", []byte("sig v1")) -} - -// TestArchHandler_RecoveredSignatureMatchesServedDatabase covers a signature -// whose fetch failed while the database was served. If the upstream has moved -// on by the time pacman asks for the signature, the new one cannot verify the -// database pacman holds, so the proxy must fail rather than serve it. -func TestArchHandler_RecoveredSignatureMatchesServedDatabase(t *testing.T) { - const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" - upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) - upstream.broken[sigPath] = true - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v1", []byte("db v1")) - - upstream.mu.Lock() - upstream.files[dbPath] = []byte("db v2") - upstream.files[sigPath] = []byte("sig v2") - upstream.broken[sigPath] = false - upstream.mu.Unlock() - - if w := serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil); w.Code != http.StatusBadGateway { - t.Errorf("signature for db v1: status = %d, body = %q, want 502", w.Code, w.Body.String()) - } - - // The next sync gets the new pair. - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v2", []byte("db v2")) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v2", []byte("sig v2")) -} - -// TestArchHandler_RecoveredSignatureForUnchangedDatabase covers the same -// failed signature fetch when the database has not changed: the signature is -// fetched on request and served. -func TestArchHandler_RecoveredSignatureForUnchangedDatabase(t *testing.T) { - const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" - upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) - upstream.broken[sigPath] = true - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v1", []byte("db v1")) - upstream.mu.Lock() - upstream.broken[sigPath] = false - upstream.mu.Unlock() - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v1", []byte("sig v1")) -} - -// TestArchHandler_SignatureSurvivesUnchangedRefresh covers a database refresh -// that returns the same bytes: the cached signature still matches and must be -// served offline afterwards. -func TestArchHandler_SignatureSurvivesUnchangedRefresh(t *testing.T) { - const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" - upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db", []byte("db v1")) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig", []byte("sig v1")) - - ageMetadata(t, proxy, h, "custom", upstream.URL, dbPath[1:]) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db refreshed unchanged", []byte("db v1")) - - upstream.available.Store(false) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig offline", []byte("sig v1")) - if got := upstream.hitCount(sigPath); got != 1 { - t.Errorf("signature upstream hits = %d, want 1", got) - } -} - -// TestArchHandler_DatabaseReplacedDuringRefresh covers a mirror that swaps -// database and signature between the proxy's two fetches. The proxy must -// notice and store the new pair, not the old database with the new signature. -func TestArchHandler_DatabaseReplacedDuringRefresh(t *testing.T) { - const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" - upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) - var swapped atomic.Bool - upstream.served = func(path string) { - if path == sigPath && swapped.CompareAndSwap(false, true) { - upstream.set(dbPath, []byte("db v2")) - upstream.set(sigPath, []byte("sig v2")) - } - } - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db", []byte("db v2")) - upstream.available.Store(false) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig", []byte("sig v2")) -} - -// TestArchHandler_RejectsQueryAndFragmentInPath covers a decoded '?' or '#' -// in the path, which would otherwise fetch one file and cache it under the -// name that follows. -func TestArchHandler_RejectsQueryAndFragmentInPath(t *testing.T) { - upstream := newArchUpstream(t, map[string][]byte{}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) - - for _, target := range []string{ - "/archlinux/core/os/x86_64/bash-5.2-1-x86_64.pkg.tar.zst%3F/foo-9-9-any.pkg.tar.zst", - "/archlinux/core/os/x86_64/bash-5.2-1-x86_64.pkg.tar.zst%23/foo-9-9-any.pkg.tar.zst", - } { - if w := serveArchRequest(h, http.MethodGet, target, nil); w.Code != http.StatusBadRequest { - t.Errorf("%s: status = %d, want 400", target, w.Code) - } - } - upstream.mu.Lock() - defer upstream.mu.Unlock() - if len(upstream.hits) != 0 { - t.Errorf("upstream was asked for %v", upstream.hits) - } -} - -// ageMetadata moves a metadata entry's fetch time past the one-hour TTL. -func ageMetadata(t *testing.T, proxy *Proxy, h *ArchHandler, repository, upstreamURL, path string) { - t.Helper() - entry, err := proxy.DB.GetMetadataCache(archEcosystem, h.metadataCacheKey(repository, upstreamURL, path)) - if err != nil || entry == nil { - t.Fatalf("metadata entry for %s: %v", path, err) - } - entry.FetchedAt.Time = time.Now().Add(-2 * time.Hour) - if err := proxy.DB.UpsertMetadataCache(entry); err != nil { - t.Fatalf("aging metadata entry: %v", err) - } -} - -// TestArchHandler_PackagesAndSignatures covers package and signature -// downloads, cache hits, offline reads and epoch filenames (both as pacman -// sends them and percent-encoded). -func TestArchHandler_PackagesAndSignatures(t *testing.T) { - files := map[string][]byte{ - "/core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst": []byte("bash package"), - "/core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst.sig": []byte("bash signature"), - "/core/os/x86_64/lz4-1:1.10.0-2-x86_64.pkg.tar.zst": []byte("lz4 package"), - "/core/os/x86_64/lz4-1:1.10.0-2-x86_64.pkg.tar.zst.sig": []byte("lz4 signature"), - } - upstream := newArchUpstream(t, files) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) - - for path, want := range files { - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), path, want) - } - - upstream.available.Store(false) - for path, want := range files { - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), path+" offline", want) - if got := upstream.hitCount(path); got != 1 { - t.Errorf("%s: upstream hits = %d, want 1", path, got) - } - } - assertArchBody(t, serveArchRequest(h, http.MethodGet, - "/archlinux/core/os/x86_64/lz4-1%3A1.10.0-2-x86_64.pkg.tar.zst", nil), "percent-encoded epoch", files["/core/os/x86_64/lz4-1:1.10.0-2-x86_64.pkg.tar.zst"]) -} - -// TestArchHandler_ArchLinuxARMLayout covers the Arch Linux ARM mirror layout -// ($arch/$repo), served through the same route. -func TestArchHandler_ArchLinuxARMLayout(t *testing.T) { - files := map[string][]byte{ - "/aarch64/core/core.db": []byte("alarm core db"), - "/aarch64/core/bash-5.2.037-1-aarch64.pkg.tar.xz": []byte("alarm bash"), - "/aarch64/core/bash-5.2.037-1-aarch64.pkg.tar.xz.sig": []byte("alarm bash signature"), - } - upstream := newArchUpstream(t, files) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinuxarm": upstream.URL}) - - for path, want := range files { - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinuxarm"+path, nil), path, want) - } - upstream.available.Store(false) - for path, want := range files { - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinuxarm"+path, nil), path+" offline", want) - } -} - -// TestArchHandler_RepositoriesDoNotShareCache checks that two repositories -// with the same path keep their own bytes, for databases and packages. -func TestArchHandler_RepositoriesDoNotShareCache(t *testing.T) { - const dbPath, pkgPath = "/core/os/x86_64/core.db", "/core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst" - a := newArchUpstream(t, map[string][]byte{dbPath: []byte("db A"), pkgPath: []byte("pkg A")}) - b := newArchUpstream(t, map[string][]byte{dbPath: []byte("db B"), pkgPath: []byte("pkg B")}) - proxy := newArchTestProxy(t, a) - proxy.HTTPClient = http.DefaultClient - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(http.DefaultClient), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"a": a.URL, "b": b.URL}) - - for _, repo := range []string{"a", "b"} { - suffix := map[string]string{"a": "A", "b": "B"}[repo] - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/"+repo+dbPath, nil), repo+" db", []byte("db "+suffix)) - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/"+repo+pkgPath, nil), repo+" pkg", []byte("pkg "+suffix)) - } -} - -// TestArchHandler_PackageHead covers HEAD: headers without a body. Range -// requests (pacman resuming a .part download) go through ServeArtifactRequest -// and are covered by its tests on seekable storage. -func TestArchHandler_PackageHead(t *testing.T) { - const path = "/core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst" - pkg := []byte("0123456789 package bytes") - upstream := newArchUpstream(t, map[string][]byte{path: pkg}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), "seed", pkg) - - w := serveArchRequest(h, http.MethodHead, "/archlinux"+path, nil) - if w.Code != http.StatusOK || w.Body.Len() != 0 { - t.Errorf("HEAD: status = %d, body length = %d, want 200 and 0", w.Code, w.Body.Len()) - } - if got := w.Header().Get("Content-Length"); got != fmt.Sprint(len(pkg)) { - t.Errorf("HEAD Content-Length = %q, want %d", got, len(pkg)) - } -} - -// TestArchHandler_PackageDownloadSendsUpstreamAuth covers upstream.auth for -// package downloads; databases are covered in -// TestArchHandler_DatabasesServedUnchanged. -func TestArchHandler_PackageDownloadSendsUpstreamAuth(t *testing.T) { - const path = "/private/os/x86_64/tool-1.0-1-x86_64.pkg.tar.zst" - upstream := newArchUpstream(t, map[string][]byte{path: []byte("private package")}) - proxy := newArchTestProxy(t, upstream) - client := upstream.Client() - client.Transport = &authRoundTripper{base: client.Transport, header: "Authorization", value: "Bearer arch-token"} - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"private": upstream.URL}) - - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/private"+path, nil), path, []byte("private package")) - if got, _ := upstream.auth.Load().(string); got != "Bearer arch-token" { - t.Errorf("Authorization = %q, want %q", got, "Bearer arch-token") - } -} - -// TestArchHandler_OtherFilesProxiedUncached covers files that are neither -// databases nor packages, such as lastupdate. -func TestArchHandler_OtherFilesProxiedUncached(t *testing.T) { - upstream := newArchUpstream(t, map[string][]byte{"/lastupdate": []byte("1760000000")}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) - - for i := range 2 { - assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/lastupdate", nil), fmt.Sprint("lastupdate ", i), []byte("1760000000")) - } - if got := upstream.hitCount("/lastupdate"); got != 2 { - t.Errorf("upstream hits = %d, want 2 (not cached)", got) - } -} - -func serveArchRequest(h *ArchHandler, method, target string, header http.Header) *httptest.ResponseRecorder { - req := httptest.NewRequest(method, target, nil) - for k, v := range header { - req.Header[k] = v - } - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - return w -} diff --git a/internal/handler/artifact_size_test.go b/internal/handler/artifact_size_test.go deleted file mode 100644 index 68dc25d..0000000 --- a/internal/handler/artifact_size_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "net/http" - "net/http/httptest" - "testing" - - "github.com/git-pkgs/registries/fetch" -) - -func artifactSizeFetcher(t *testing.T) (*fetch.Fetcher, string) { - t.Helper() - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.(http.Flusher).Flush() - _, _ = io.WriteString(w, "0123456789") - })) - t.Cleanup(upstream.Close) - f := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxArtifactSize(4)) - t.Cleanup(func() { _ = f.Close() }) - return f, upstream.URL -} - -func TestOversizedArtifactIsNotCached(t *testing.T) { - proxy, db, store, _ := setupTestProxy(t) - limited, upstream := artifactSizeFetcher(t) - proxy.Fetcher = limited - - _, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "big", "1.0.0", "big-1.0.0.tgz", upstream+"/big-1.0.0.tgz") - if !errors.Is(err, fetch.ErrArtifactTooLarge) { - t.Fatalf("GetOrFetchArtifactFromURL error = %v, want fetch.ErrArtifactTooLarge", err) - } - if len(store.files) != 0 { - t.Errorf("storage holds %d objects after an oversized fetch, want 0", len(store.files)) - } - fetchErr := err - record, err := db.GetArtifact("pkg:npm/big@1.0.0", "big-1.0.0.tgz") - if err != nil { - t.Fatalf("GetArtifact failed: %v", err) - } - if record != nil { - t.Error("oversized artifact was recorded in the cache database") - } - - rec := httptest.NewRecorder() - proxy.serveArtifactError(rec, fetchErr, "failed to fetch package") - if rec.Code != http.StatusBadGateway { - t.Errorf("status = %d, want %d", rec.Code, http.StatusBadGateway) - } -} - -func TestOversizedStreamedArtifactFailsTheStream(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.StreamArtifacts = true - limited, upstream := artifactSizeFetcher(t) - proxy.Fetcher = limited - - result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "big", "1.0.0", "big-1.0.0.tgz", upstream+"/big-1.0.0.tgz") - if err != nil { - t.Fatalf("GetOrFetchArtifactFromURL failed: %v", err) - } - defer func() { _ = result.Reader.Close() }() - if _, err := io.ReadAll(result.Reader); !errors.Is(err, fetch.ErrArtifactTooLarge) { - t.Errorf("streamed body error = %v, want fetch.ErrArtifactTooLarge", err) - } -} diff --git a/internal/handler/cache_key.go b/internal/handler/cache_key.go deleted file mode 100644 index 6c2e8a5..0000000 --- a/internal/handler/cache_key.go +++ /dev/null @@ -1,21 +0,0 @@ -package handler - -import ( - "crypto/sha256" - "encoding/hex" - "strconv" -) - -// hashedCacheKey derives a metadata cache key from parts such as the -// upstream URL, the request path and the query. Each part is length-prefixed -// before hashing, so no two different part lists share a key, whatever bytes -// the parts contain. -func hashedCacheKey(parts ...string) string { - h := sha256.New() - for _, part := range parts { - _, _ = h.Write([]byte(strconv.Itoa(len(part)))) - _, _ = h.Write([]byte{':'}) - _, _ = h.Write([]byte(part)) - } - return hex.EncodeToString(h.Sum(nil)) -} diff --git a/internal/handler/cache_key_test.go b/internal/handler/cache_key_test.go deleted file mode 100644 index 8af47a2..0000000 --- a/internal/handler/cache_key_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package handler - -import ( - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" -) - -// TestConanMetadataKeysKeepReferencesApart checks that Conan references whose -// paths only differ in where '/' and '_' fall, or in a query, are cached -// apart. Names and versions may contain '_'. -func TestConanMetadataKeysKeepReferencesApart(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, r.URL.Path+"?"+r.URL.RawQuery) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - srv := httptest.NewServer(NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes()) - defer srv.Close() - - for _, pair := range [][2]string{ - {"/v2/conans/foo/bar_1.0/_/_/latest", "/v2/conans/foo_bar/1.0/_/_/latest"}, - {"/v2/conans/foo/bar_1.0/_/_/revisions/r1/files/conanfile.py", "/v2/conans/foo_bar/1.0/_/_/revisions/r1/files/conanfile.py"}, - {"/v2/conans/search?q=zlib", "/v2/conans/search_q=zlib"}, - } { - for _, target := range pair { - resp, err := http.Get(srv.URL + target) - if err != nil { - t.Fatal(err) - } - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - want := target - if !strings.Contains(want, "?") { - want += "?" - } - if string(body) != want { - t.Errorf("GET %s served %q, want %q", target, body, want) - } - } - } -} - -// TestMetadataCacheKeysUnambiguous checks that the path-based metadata cache -// keys of the other ecosystems keep paths apart that differ only in where -// '/' and '_' fall. -func TestMetadataCacheKeysUnambiguous(t *testing.T) { - const a, b = "x/foo/bar_1.0/file", "x/foo_bar/1.0/file" - - maven := mavenSource{upstreamURL: "https://repo.test"} - debian := NewDebianHandler(nil, "http://proxy.test", "https://deb.test", nil) - rpm := NewRPMHandlerWithUpstream(nil, "http://proxy.test", "https://rpm.test") - cran := NewCRANHandlerWithUpstream(nil, "http://proxy.test", "https://cran.test") - conda := NewCondaHandlerWithUpstream(nil, "http://proxy.test", "https://conda.test") - - for name, key := range map[string]func(string) string{ - "maven": maven.metadataCacheKey, - "debian": func(p string) string { return debian.metadataCacheKeyFor("", debian.upstreamURL, p) }, - "rpm": func(p string) string { return rpm.metadataCacheKey(p) }, - "cran": func(p string) string { return cran.metadataCacheKey("/" + p) }, - "conda": func(p string) string { return conda.metadataCacheKey("/" + p) }, - } { - if key(a) == key(b) { - t.Errorf("%s: %s and %s share metadata cache key %q", name, a, b, key(a)) - } - } -} diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index aca7416..5d7810c 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -2,16 +2,14 @@ package handler import ( "bufio" - "bytes" "encoding/json" "errors" "fmt" "net/http" - "net/url" "strings" "time" - "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -23,14 +21,6 @@ const ( cargoIndexLen3 = 3 ) -// Crates are stored under pkg:cargo/{name}, with the name cargo puts in the -// download URL, which is the name as published. purl keeps cargo names as -// they are, so package rules use the default key mapping and have to match -// that name exactly. -func init() { - retention.Default.Register(retention.Spec{Key: "cargo", CanonicalPackage: retention.DefaultCanonical("cargo")}) -} - // CargoHandler handles cargo registry protocol requests. type CargoHandler struct { proxy *Proxy @@ -40,18 +30,11 @@ type CargoHandler struct { } // NewCargoHandler creates a new cargo protocol handler. -func NewCargoHandler(proxy *Proxy, proxyURL, indexURL, downloadURL string) *CargoHandler { - if strings.TrimSpace(indexURL) == "" { - indexURL = cargoUpstream - } - if strings.TrimSpace(downloadURL) == "" { - downloadURL = cargoDownloadBase - } - +func NewCargoHandler(proxy *Proxy, proxyURL string) *CargoHandler { return &CargoHandler{ proxy: proxy, - indexURL: strings.TrimSuffix(indexURL, "/"), - downloadURL: strings.TrimSuffix(downloadURL, "/"), + indexURL: cargoUpstream, + downloadURL: cargoDownloadBase, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } @@ -91,7 +74,7 @@ func (h *CargoHandler) handleConfig(w http.ResponseWriter, r *http.Request) { DL: h.proxyURL + "/cargo/crates/{crate}/{version}/download", } - w.Header().Set(headerContentType, "application/json") + w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(config) } @@ -103,7 +86,7 @@ func (h *CargoHandler) handleIndex(w http.ResponseWriter, r *http.Request) { return } - h.proxy.Logger.Debug("cargo index request", "crate", name) + h.proxy.Logger.Info("cargo index request", "crate", name) indexPath := h.buildIndexPath(name) upstreamURL := fmt.Sprintf("%s/%s", h.indexURL, indexPath) @@ -114,9 +97,6 @@ func (h *CargoHandler) handleIndex(w http.ResponseWriter, r *http.Request) { http.Error(w, "not found", http.StatusNotFound) return } - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("failed to fetch upstream index", "error", err) http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return @@ -126,7 +106,7 @@ func (h *CargoHandler) handleIndex(w http.ResponseWriter, r *http.Request) { contentType = "text/plain; charset=utf-8" } - w.Header().Set(headerContentType, contentType) + w.Header().Set("Content-Type", contentType) w.WriteHeader(http.StatusOK) h.applyCooldownFiltering(w, body) } @@ -138,16 +118,12 @@ type crateIndexEntry struct { } func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWriter, body []byte) { - cooldownEnabled := h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() - if !cooldownEnabled && h.proxy.Denylist == nil { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { _, _ = downstreamResponse.Write(body) return } scanner := bufio.NewScanner(strings.NewReader(string(body))) - // The response is already size-bounded by FetchOrCacheMetadata. Crate - // entries with many dependencies can exceed Scanner's default token limit. - scanner.Buffer(nil, len(body)+1) for scanner.Scan() { line := scanner.Text() @@ -160,19 +136,16 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr continue } - if h.proxy.versionDenied("cargo", crate.Name, crate.Version) { - continue - } publishedAt, err := time.Parse(time.RFC3339, crate.PublishTime) - if !cooldownEnabled || crate.PublishTime == "" || err != nil { + if crate.PublishTime == "" || err != nil { _, _ = downstreamResponse.Write([]byte(line + "\n")) continue } - cratePURL := canonicalPackagePURL("cargo", crate.Name) + cratePURL := purl.MakePURLString("cargo", crate.Name, "") - if !h.proxy.cooldown().IsAllowed("cargo", cratePURL, publishedAt) { + if !h.proxy.Cooldown.IsAllowed("cargo", cratePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering cargo version", "crate", crate.Name, "version", crate.Version, "published", crate.PublishTime) @@ -187,32 +160,6 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr } } -// publishTime reads a crate version's pubtime from the sparse index, sharing -// the index cache with handleIndex. Entries published before crates.io -// recorded pubtime have none. -func (h *CargoHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { - upstreamURL := fmt.Sprintf("%s/%s", h.indexURL, h.buildIndexPath(name)) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "cargo", name, upstreamURL, "text/plain") - if err != nil { - return time.Time{}, false, err - } - for line := range bytes.SplitSeq(body, []byte("\n")) { - if len(bytes.TrimSpace(line)) == 0 { - continue - } - var entry crateIndexEntry - if err := json.Unmarshal(line, &entry); err != nil { - return time.Time{}, false, err - } - if entry.Version != version { - continue - } - published, err := time.Parse(time.RFC3339, entry.PublishTime) - return published, err == nil, nil - } - return time.Time{}, false, nil -} - // buildIndexPath builds the sparse index path for a crate name. func (h *CargoHandler) buildIndexPath(name string) string { name = strings.ToLower(name) @@ -241,32 +188,15 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { filename := fmt.Sprintf("%s-%s.crate", name, version) - h.proxy.Logger.Debug("cargo download request", + h.proxy.Logger.Info("cargo download request", "crate", name, "version", version, "filename", filename) - if h.proxy.withholdInCooldown(w, r, downloadVersion{ - ecosystem: "cargo", policyName: name, storedName: name, version: version, cacheName: name, filename: filename, - lookup: func() (time.Time, bool, bool, error) { - published, ok, err := h.publishTime(r, name, version) - return published, ok, false, err - }, - }) { - return - } - - downloadURL := fmt.Sprintf( - "%s/%s/%s", - h.downloadURL, - url.PathEscape(name), - url.PathEscape(filename), - ) - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), "cargo", name, version, filename, downloadURL, - ) + result, err := h.proxy.GetOrFetchArtifact(r.Context(), "cargo", name, version, filename) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch crate") + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch crate", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) + ServeArtifact(w, result) } diff --git a/internal/handler/cargo_test.go b/internal/handler/cargo_test.go index de3b2f3..10d3faf 100644 --- a/internal/handler/cargo_test.go +++ b/internal/handler/cargo_test.go @@ -2,7 +2,6 @@ package handler import ( "encoding/json" - "io" "log/slog" "net/http" "net/http/httptest" @@ -11,7 +10,6 @@ import ( "time" "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/registries/fetch" ) func cargoTestProxy() *Proxy { @@ -72,73 +70,6 @@ func TestCargoConfigEndpoint(t *testing.T) { } } -func TestCargoHandlerUsesConfiguredUpstreams(t *testing.T) { - t.Run("index", func(t *testing.T) { - var requestPath, authHeader string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requestPath = r.URL.Path - authHeader = r.Header.Get("Authorization") - if authHeader != "Bearer cargo-token" { - w.WriteHeader(http.StatusUnauthorized) - return - } - w.Header().Set("Content-Type", "text/plain") - _, _ = io.WriteString(w, `{"name":"serde","vers":"1.0.0"}`) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - authorizeUpstream(proxy, "Bearer cargo-token") - h := NewCargoHandler( - proxy, - "http://proxy.test", - upstream.URL+"/index/", - "https://crates.example.test/files/", - ) - - req := httptest.NewRequest(http.MethodGet, "/se/rd/serde", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if requestPath != "/index/se/rd/serde" { - t.Errorf("upstream path = %q, want %q", requestPath, "/index/se/rd/serde") - } - if authHeader != "Bearer cargo-token" { - t.Errorf("Authorization = %q, want %q", authHeader, "Bearer cargo-token") - } - }) - - t.Run("download", func(t *testing.T) { - proxy, _, _, artifactFetcher := setupTestProxy(t) - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("crate")), - ContentType: "application/gzip", - } - h := NewCargoHandler( - proxy, - "http://proxy.test", - "https://index.example.test/root/", - "https://crates.example.test/files/", - ) - - req := httptest.NewRequest(http.MethodGet, "/crates/serde/1.0.0/download", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - want := "https://crates.example.test/files/serde/serde-1.0.0.crate" - if artifactFetcher.fetchedURL != want { - t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) - } - }) -} - func TestCargoIndexProxy(t *testing.T) { // Create a mock upstream index server indexContent := `{"name":"serde","vers":"1.0.0","deps":[],"cksum":"abc123"} diff --git a/internal/handler/chef.go b/internal/handler/chef.go deleted file mode 100644 index cf2002d..0000000 --- a/internal/handler/chef.go +++ /dev/null @@ -1,355 +0,0 @@ -package handler - -import ( - "bytes" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "path" - "regexp" - "strings" -) - -const ( - chefEcosystem = "chef" - chefMountPath = "/chef" - chefAPIPath = "/api/v1" - // chefUpstreamKeyLength is how much of the upstream URL's digest scopes - // metadata cache keys and archive filenames: enough to keep two upstreams - // apart. - chefUpstreamKeyLength = 16 -) - -var ( - // chefCookbookName matches Supermarket cookbook names: letters, digits, - // hyphens, underscores and dots, not starting with a dot. - chefCookbookName = regexp.MustCompile(`^[A-Za-z0-9_-][A-Za-z0-9_.-]*$`) - // chefVersion matches Chef cookbook versions, which have two or three - // numeric parts. - chefVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+(\.[0-9]+)?$`) - - errChefDocument = errors.New("unexpected Chef Supermarket document") -) - -// ChefHandler serves a read-only Chef Supermarket at /chef. Berkshelf and -// Policyfiles resolve dependencies against /universe and then download each -// cookbook through the cookbook API. Every download location in those -// documents is rewritten to the proxy, so cookbooks are fetched through it and -// cached; dependency constraints and every other field pass through unchanged. -type ChefHandler struct { - proxy *Proxy - proxyURL string - upstreamURL string - // upstreamKey identifies the upstream in cache keys, so pointing - // upstream.chef at another Supermarket does not serve the previous one's - // documents or archives. - upstreamKey string -} - -// NewChefHandler creates a Chef Supermarket handler for the Supermarket at -// upstreamURL, such as https://supermarket.chef.io or a private Supermarket. -func NewChefHandler(proxy *Proxy, proxyURL, upstreamURL string) *ChefHandler { - upstreamURL = strings.TrimSuffix(upstreamURL, "/") - digest := sha256.Sum256([]byte(upstreamURL)) - return &ChefHandler{ - proxy: proxy, - proxyURL: strings.TrimSuffix(proxyURL, "/"), - upstreamURL: upstreamURL, - upstreamKey: hex.EncodeToString(digest[:])[:chefUpstreamKeyLength], - } -} - -// Routes returns the HTTP handler for Chef Supermarket requests. -func (h *ChefHandler) Routes() http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("GET /universe", h.handleUniverse) - mux.HandleFunc("GET "+chefAPIPath+"/cookbooks/{name}", h.handleCookbook) - mux.HandleFunc("GET "+chefAPIPath+"/cookbooks/{name}/versions/{version}", h.handleVersion) - mux.HandleFunc("GET "+chefAPIPath+"/cookbooks/{name}/versions/{version}/download", h.handleDownload) - return mux -} - -func (h *ChefHandler) handleUniverse(w http.ResponseWriter, r *http.Request) { - h.serveDocument(w, r, "universe", h.upstreamURL+"/universe", h.rewriteUniverse) -} - -func (h *ChefHandler) handleCookbook(w http.ResponseWriter, r *http.Request) { - name, ok := chefCookbookFromRequest(r) - if !ok { - http.Error(w, "invalid cookbook name", http.StatusBadRequest) - return - } - h.serveDocument(w, r, "cookbooks/"+name, h.upstreamCookbookURL(name), func(body []byte) ([]byte, error) { - return h.rewriteCookbook(name, body) - }) -} - -func (h *ChefHandler) handleVersion(w http.ResponseWriter, r *http.Request) { - name, version, ok := chefVersionFromRequest(r) - if !ok { - http.Error(w, "invalid cookbook version", http.StatusBadRequest) - return - } - h.serveDocument(w, r, "cookbooks/"+name+"/versions/"+version, h.upstreamVersionURL(name, version), func(body []byte) ([]byte, error) { - return h.rewriteVersion(name, version, body) - }) -} - -// handleDownload serves a cookbook archive from the cache. An archive not -// cached yet is fetched through the Supermarket download endpoint, following -// the redirect it answers with. A cached archive is served without the -// upstream. Archives are cached per upstream, since two Supermarkets can -// publish different cookbooks under the same name and version. -func (h *ChefHandler) handleDownload(w http.ResponseWriter, r *http.Request) { - name, version, ok := chefVersionFromRequest(r) - if !ok { - http.Error(w, "invalid cookbook version", http.StatusBadRequest) - return - } - - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), chefEcosystem, name, version, h.archiveFilename(name, version), h.upstreamVersionURL(name, version)+"/download") - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch cookbook") - return - } - if result.Artifact.MediaType == "" { - w.Header().Set(headerContentType, "application/gzip") - } - ServeArtifactRequest(w, r, result) -} - -// serveDocument serves a JSON document from the metadata cache, which applies -// the metadata TTL and falls back to the stored copy while the upstream is -// unavailable, rewritten by rewrite. -func (h *ChefHandler) serveDocument(w http.ResponseWriter, r *http.Request, documentKey, upstreamURL string, rewrite func([]byte) ([]byte, error)) { - cacheKey := h.cacheKey(documentKey) - if stored, ok := h.proxy.storedRewrite(chefEcosystem, cacheKey, h.proxyURL, cacheKey); ok { - serveRewrittenMetadata(w, r, stored) - return - } - - upstream := h.proxy.fetchMetadataDocument(r.Context(), chefEcosystem, cacheKey, upstreamURL, contentTypeJSON, nil) - if err := upstream.err; err != nil { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("Chef Supermarket request failed", "url", upstreamURL, "error", err) - http.Error(w, "upstream request failed", http.StatusBadGateway) - return - } - - rewritten, err := h.proxy.cachedRewrite(r.Context(), chefEcosystem, h.proxyURL, cacheKey, upstream.body, upstream.digest, - func(body []byte) ([]byte, []string, error) { - out, err := rewrite(body) - return out, nil, err - }) - if err != nil { - if r.Context().Err() != nil { - return // the client left while waiting on a shared rewrite - } - // Serving the document unrewritten would send clients straight to - // the upstream, bypassing the cache. - h.proxy.Logger.Warn("failed to rewrite Chef Supermarket document", "url", upstreamURL, "error", err) - http.Error(w, "invalid Chef Supermarket response", http.StatusBadGateway) - return - } - markStale(w, upstream.stale) - serveRewrittenMetadata(w, r, rewritten) -} - -// rewriteUniverse points every version's location_path and download_url at -// the proxy. The universe maps cookbook names to versions to their location -// and dependencies; dependencies and any other field are copied as they are. -func (h *ChefHandler) rewriteUniverse(body []byte) ([]byte, error) { - var universe map[string]map[string]map[string]json.RawMessage - if err := json.Unmarshal(body, &universe); err != nil { - return nil, fmt.Errorf("%w: universe: %w", errChefDocument, err) - } - - locationPath, err := jsonString(h.apiURL()) - if err != nil { - return nil, err - } - for name, versions := range universe { - for version, entry := range versions { - if entry == nil { - continue - } - downloadURL, err := jsonString(h.downloadURL(name, version)) - if err != nil { - return nil, err - } - entry["location_path"] = locationPath - entry["download_url"] = downloadURL - } - } - return encodeChefJSON(universe) -} - -// rewriteCookbook points a cookbook's version URLs at the proxy. -func (h *ChefHandler) rewriteCookbook(name string, body []byte) ([]byte, error) { - var cookbook map[string]json.RawMessage - if err := json.Unmarshal(body, &cookbook); err != nil { - return nil, fmt.Errorf("%w: cookbook %s: %w", errChefDocument, name, err) - } - - if raw, ok := cookbook["latest_version"]; ok { - rewritten, err := h.rewriteVersionURL(name, raw) - if err != nil { - return nil, err - } - cookbook["latest_version"] = rewritten - } - if raw, ok := cookbook["versions"]; ok { - var versions []json.RawMessage - if err := json.Unmarshal(raw, &versions); err != nil { - return nil, fmt.Errorf("%w: cookbook %s versions: %w", errChefDocument, name, err) - } - for i, versionURL := range versions { - rewritten, err := h.rewriteVersionURL(name, versionURL) - if err != nil { - return nil, err - } - versions[i] = rewritten - } - encoded, err := encodeChefJSON(versions) - if err != nil { - return nil, err - } - cookbook["versions"] = encoded - } - return encodeChefJSON(cookbook) -} - -// rewriteVersion points a cookbook version's cookbook and file URLs at the -// proxy. -func (h *ChefHandler) rewriteVersion(name, version string, body []byte) ([]byte, error) { - var document map[string]json.RawMessage - if err := json.Unmarshal(body, &document); err != nil { - return nil, fmt.Errorf("%w: cookbook %s version %s: %w", errChefDocument, name, version, err) - } - - if _, ok := document["cookbook"]; ok { - cookbookURL, err := jsonString(h.cookbookURL(name)) - if err != nil { - return nil, err - } - document["cookbook"] = cookbookURL - } - if _, ok := document["file"]; ok { - fileURL, err := jsonString(h.downloadURL(name, version)) - if err != nil { - return nil, err - } - document["file"] = fileURL - } - return encodeChefJSON(document) -} - -// rewriteVersionURL rewrites an upstream cookbook version URL, a JSON string -// ending in /versions/{version}, to the proxy. Anything else, such as null, -// is returned unchanged. -func (h *ChefHandler) rewriteVersionURL(name string, raw json.RawMessage) (json.RawMessage, error) { - var versionURL string - if err := json.Unmarshal(raw, &versionURL); err != nil { - return raw, nil //nolint:nilerr // not a string, so not a URL to rewrite - } - parsed, err := url.Parse(versionURL) - if err != nil { - return raw, nil //nolint:nilerr // not a URL, so nothing to rewrite - } - version, ok := normalizeChefVersion(path.Base(parsed.Path)) - if !ok { - return raw, nil - } - return jsonString(h.versionURL(name, version)) -} - -func (h *ChefHandler) apiURL() string { - return h.proxyURL + chefMountPath + chefAPIPath -} - -func (h *ChefHandler) cookbookURL(name string) string { - return h.apiURL() + "/cookbooks/" + url.PathEscape(name) -} - -func (h *ChefHandler) versionURL(name, version string) string { - return h.cookbookURL(name) + "/versions/" + url.PathEscape(version) -} - -func (h *ChefHandler) downloadURL(name, version string) string { - return h.versionURL(name, version) + "/download" -} - -func (h *ChefHandler) upstreamCookbookURL(name string) string { - return h.upstreamURL + chefAPIPath + "/cookbooks/" + url.PathEscape(name) -} - -func (h *ChefHandler) upstreamVersionURL(name, version string) string { - return h.upstreamCookbookURL(name) + "/versions/" + url.PathEscape(version) -} - -// cacheKey prefixes a metadata cache key with the upstream's key. -func (h *ChefHandler) cacheKey(documentKey string) string { - return h.upstreamKey + "/" + documentKey -} - -// archiveFilename names a cookbook archive in the cache. The filename is part -// of the artifact's cache identity, so it carries the upstream's key; the -// .tgz extension is kept for browsing the archive. -func (h *ChefHandler) archiveFilename(name, version string) string { - return name + "-" + version + "-" + h.upstreamKey + ".tgz" -} - -func chefCookbookFromRequest(r *http.Request) (string, bool) { - name := r.PathValue("name") - if !chefCookbookName.MatchString(name) || containsPathTraversal(name) { - return "", false - } - return name, true -} - -func chefVersionFromRequest(r *http.Request) (name, version string, ok bool) { - name, ok = chefCookbookFromRequest(r) - if !ok { - return "", "", false - } - version, ok = normalizeChefVersion(r.PathValue("version")) - return name, version, ok -} - -// normalizeChefVersion accepts a version in either form Supermarket serves -// (1.2.3, or 1_2_3 as Berkshelf writes it in URLs) and returns the dotted -// form. -func normalizeChefVersion(version string) (string, bool) { - version = strings.ReplaceAll(version, "_", ".") - if !chefVersion.MatchString(version) { - return "", false - } - return version, true -} - -func jsonString(s string) (json.RawMessage, error) { - return encodeChefJSON(s) -} - -// encodeChefJSON encodes v without escaping HTML characters, so dependency -// constraints such as ">= 1.0" keep their upstream spelling. -func encodeChefJSON(v any) (json.RawMessage, error) { - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.SetEscapeHTML(false) - if err := enc.Encode(v); err != nil { - return nil, err - } - return bytes.TrimSuffix(buf.Bytes(), []byte("\n")), nil -} diff --git a/internal/handler/chef_test.go b/internal/handler/chef_test.go deleted file mode 100644 index 51290a5..0000000 --- a/internal/handler/chef_test.go +++ /dev/null @@ -1,475 +0,0 @@ -package handler - -import ( - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" - "github.com/git-pkgs/registries/fetch" -) - -const ( - chefTestProxyURL = "http://proxy.example" - chefTestAPI = chefTestProxyURL + "/chef/api/v1" - chefTestTarball = "cookbook tarball bytes" -) - -// fakeSupermarket serves the parts of the Supermarket API Berkshelf uses, in -// the shapes supermarket.chef.io returns them. Downloads redirect to a second -// server standing in for the storage host, as Supermarket redirects to S3. -type fakeSupermarket struct { - api *httptest.Server - files *httptest.Server - // filesURL is files.URL on another hostname, so a redirect to it leaves - // the API host the way a redirect to S3 does. - filesURL string - - available atomic.Bool - // token, when set, is the bearer token the API requires. - token string - // tarball is the content of every cookbook archive it serves. - tarball string - - mu sync.Mutex - requests map[string]int - fileAuth []string -} - -func newFakeSupermarket(t *testing.T, token string) *fakeSupermarket { - t.Helper() - s := &fakeSupermarket{token: token, tarball: chefTestTarball, requests: map[string]int{}} - s.available.Store(true) - - s.files = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - s.requests["files "+r.URL.Path]++ - s.fileAuth = append(s.fileAuth, r.Header.Get("Authorization")) - s.mu.Unlock() - if !s.available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - if r.URL.Path != "/cookbook_versions/tarballs/34899/original/apt.tgz" { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/x-gzip") - _, _ = w.Write([]byte(s.tarball)) - })) - t.Cleanup(s.files.Close) - s.filesURL = strings.Replace(s.files.URL, "127.0.0.1", "localhost", 1) - - s.api = httptest.NewServer(http.HandlerFunc(s.serveAPI)) - t.Cleanup(s.api.Close) - return s -} - -func (s *fakeSupermarket) serveAPI(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - s.requests[r.URL.Path]++ - s.mu.Unlock() - if !s.available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - if s.token != "" && r.Header.Get("Authorization") != "Bearer "+s.token { - http.Error(w, "unauthorized", http.StatusUnauthorized) - return - } - - api := s.api.URL + "/api/v1" - w.Header().Set("Content-Type", "application/json") - switch r.URL.Path { - case "/universe": - // Like Supermarket's, these locations carry an explicit port. - located := api - _, _ = fmt.Fprintf(w, `{"apt":{"7.5.22":{"location_type":"opscode","location_path":%q,"download_url":%q,"dependencies":{"compat_resource":">= 0.0.0"}},`+ - `"7.6.0":{"location_type":"opscode","location_path":%q,"download_url":%q,"dependencies":{}}},`+ - `"compat_resource":{"12.19.1":{"location_type":"opscode","location_path":%q,"download_url":%q,"dependencies":{"apt":"~> 7.5"}}}}`, - located, api+"/cookbooks/apt/versions/7.5.22/download", - located, api+"/cookbooks/apt/versions/7.6.0/download", - located, api+"/cookbooks/compat_resource/versions/12.19.1/download") - case "/api/v1/cookbooks/apt": - _, _ = fmt.Fprintf(w, `{"name":"apt","maintainer":"sous-chefs","latest_version":%q,"external_url":"https://github.com/sous-chefs/apt","deprecated":false,"versions":[%q,%q]}`, - api+"/cookbooks/apt/versions/7.6.0", api+"/cookbooks/apt/versions/7.6.0", api+"/cookbooks/apt/versions/7.5.22") - case "/api/v1/cookbooks/apt/versions/7.5.22": - _, _ = fmt.Fprintf(w, `{"license":"Apache-2.0","tarball_file_size":%d,"version":"7.5.22","cookbook":%q,"file":%q,"dependencies":{"compat_resource":">= 0.0.0"}}`, - len(s.tarball), api+"/cookbooks/apt", api+"/cookbooks/apt/versions/7.5.22/download") - case "/api/v1/cookbooks/apt/versions/7.5.22/download": - http.Redirect(w, r, s.filesURL+"/cookbook_versions/tarballs/34899/original/apt.tgz?1714639378", http.StatusFound) - default: - http.NotFound(w, r) - } -} - -func (s *fakeSupermarket) count(key string) int { - s.mu.Lock() - defer s.mu.Unlock() - return s.requests[key] -} - -// newChefTestHandler returns a handler for s using client for every upstream -// request, with metadata caching on. -func newChefTestHandler(t *testing.T, s *fakeSupermarket, client *http.Client) (*ChefHandler, *Proxy) { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = client - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - return NewChefHandler(proxy, chefTestProxyURL, s.api.URL), proxy -} - -func serveChefRequest(h *ChefHandler, target string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - return w -} - -// proxyPath turns an absolute proxy URL from a rewritten document into the -// path the handler is mounted at. -func proxyPath(t *testing.T, rawURL string) string { - t.Helper() - parsed, err := url.Parse(rawURL) - if err != nil { - t.Fatalf("parse %q: %v", rawURL, err) - } - if parsed.Scheme+"://"+parsed.Host != chefTestProxyURL || !strings.HasPrefix(parsed.Path, "/chef/") { - t.Fatalf("URL %q does not point at the proxy", rawURL) - } - return strings.TrimPrefix(parsed.Path, "/chef") -} - -type chefUniverseEntry struct { - LocationType string `json:"location_type"` - LocationPath string `json:"location_path"` - DownloadURL string `json:"download_url"` - Dependencies json.RawMessage `json:"dependencies"` -} - -func decodeChefUniverse(t *testing.T, body []byte) map[string]map[string]chefUniverseEntry { - t.Helper() - var universe map[string]map[string]chefUniverseEntry - if err := json.Unmarshal(body, &universe); err != nil { - t.Fatalf("decode universe: %v\n%s", err, body) - } - return universe -} - -func TestChefUniverseRewritesLocations(t *testing.T) { - s := newFakeSupermarket(t, "") - h, _ := newChefTestHandler(t, s, s.api.Client()) - - w := serveChefRequest(h, "/universe") - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - if got := w.Header().Get("Content-Type"); got != contentTypeJSON { - t.Errorf("Content-Type = %q, want %q", got, contentTypeJSON) - } - body := w.Body.String() - if strings.Contains(body, s.api.URL) { - t.Errorf("rewritten universe still points at the upstream: %s", body) - } - - universe := decodeChefUniverse(t, w.Body.Bytes()) - wantDeps := map[string]map[string]string{ - "apt": {"7.5.22": `{"compat_resource":">= 0.0.0"}`, "7.6.0": `{}`}, - "compat_resource": {"12.19.1": `{"apt":"~> 7.5"}`}, - } - for name, versions := range wantDeps { - for version, deps := range versions { - entry, ok := universe[name][version] - if !ok { - t.Errorf("universe lost %s %s", name, version) - continue - } - if entry.LocationType != "opscode" { - t.Errorf("%s %s location_type = %q, want opscode", name, version, entry.LocationType) - } - if entry.LocationPath != chefTestAPI { - t.Errorf("%s %s location_path = %q, want %q", name, version, entry.LocationPath, chefTestAPI) - } - wantDownload := chefTestAPI + "/cookbooks/" + name + "/versions/" + version + "/download" - if entry.DownloadURL != wantDownload { - t.Errorf("%s %s download_url = %q, want %q", name, version, entry.DownloadURL, wantDownload) - } - if string(entry.Dependencies) != deps { - t.Errorf("%s %s dependencies = %s, want %s unchanged", name, version, entry.Dependencies, deps) - } - } - } - // Constraints keep their spelling rather than being HTML-escaped. - if !strings.Contains(body, `">= 0.0.0"`) || !strings.Contains(body, `"~> 7.5"`) { - t.Errorf("dependency constraints were re-escaped: %s", body) - } -} - -func TestChefCookbookMetadataRewritesVersionURLs(t *testing.T) { - s := newFakeSupermarket(t, "") - h, _ := newChefTestHandler(t, s, s.api.Client()) - - w := serveChefRequest(h, "/api/v1/cookbooks/apt") - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - var cookbook struct { - Name string `json:"name"` - LatestVersion string `json:"latest_version"` - Versions []string `json:"versions"` - ExternalURL string `json:"external_url"` - Deprecated bool `json:"deprecated"` - } - if err := json.Unmarshal(w.Body.Bytes(), &cookbook); err != nil { - t.Fatalf("decode: %v", err) - } - if cookbook.LatestVersion != chefTestAPI+"/cookbooks/apt/versions/7.6.0" { - t.Errorf("latest_version = %q", cookbook.LatestVersion) - } - want := []string{chefTestAPI + "/cookbooks/apt/versions/7.6.0", chefTestAPI + "/cookbooks/apt/versions/7.5.22"} - if strings.Join(cookbook.Versions, ",") != strings.Join(want, ",") { - t.Errorf("versions = %v, want %v", cookbook.Versions, want) - } - if cookbook.Name != "apt" || cookbook.ExternalURL != "https://github.com/sous-chefs/apt" || cookbook.Deprecated { - t.Errorf("unrelated fields changed: %+v", cookbook) - } -} - -func TestChefVersionMetadataAcceptsBerkshelfVersions(t *testing.T) { - s := newFakeSupermarket(t, "") - h, _ := newChefTestHandler(t, s, s.api.Client()) - - // Berkshelf writes versions with underscores in cookbook API paths. - for _, version := range []string{"7_5_22", "7.5.22"} { - w := serveChefRequest(h, "/api/v1/cookbooks/apt/versions/"+version) - if w.Code != http.StatusOK { - t.Fatalf("%s: status = %d, want 200: %s", version, w.Code, w.Body.String()) - } - var document struct { - Version string `json:"version"` - Cookbook string `json:"cookbook"` - File string `json:"file"` - Dependencies json.RawMessage `json:"dependencies"` - License string `json:"license"` - } - if err := json.Unmarshal(w.Body.Bytes(), &document); err != nil { - t.Fatalf("%s: decode: %v", version, err) - } - if document.Cookbook != chefTestAPI+"/cookbooks/apt" { - t.Errorf("%s: cookbook = %q", version, document.Cookbook) - } - if document.File != chefTestAPI+"/cookbooks/apt/versions/7.5.22/download" { - t.Errorf("%s: file = %q", version, document.File) - } - if string(document.Dependencies) != `{"compat_resource":">= 0.0.0"}` || document.License != "Apache-2.0" || document.Version != "7.5.22" { - t.Errorf("%s: unrelated fields changed: %+v", version, document) - } - } - // Both spellings are one document, fetched once. - if got := s.count("/api/v1/cookbooks/apt/versions/7.5.22"); got != 1 { - t.Errorf("upstream version requests = %d, want 1", got) - } -} - -// TestChefBerkshelfFlow follows the requests Berkshelf makes: the universe, -// then the cookbook version at the universe's location_path, then its file. -// The download redirects to the storage host and is cached. It is then served -// with the upstream down, as is the universe. -func TestChefBerkshelfFlow(t *testing.T) { - s := newFakeSupermarket(t, "") - h, proxy := newChefTestHandler(t, s, s.api.Client()) - - universe := decodeChefUniverse(t, serveChefRequest(h, "/universe").Body.Bytes()) - locationPath := universe["apt"]["7.5.22"].LocationPath - - versionResponse := serveChefRequest(h, proxyPath(t, locationPath+"/cookbooks/apt/versions/7_5_22")) - if versionResponse.Code != http.StatusOK { - t.Fatalf("version status = %d: %s", versionResponse.Code, versionResponse.Body.String()) - } - var version struct { - File string `json:"file"` - } - if err := json.Unmarshal(versionResponse.Body.Bytes(), &version); err != nil { - t.Fatalf("decode version: %v", err) - } - - download := serveChefRequest(h, proxyPath(t, version.File)) - if download.Code != http.StatusOK { - t.Fatalf("download status = %d: %s", download.Code, download.Body.String()) - } - if download.Body.String() != chefTestTarball { - t.Errorf("download body = %q, want the tarball", download.Body.String()) - } - if got := s.count("files /cookbook_versions/tarballs/34899/original/apt.tgz"); got != 1 { - t.Errorf("storage host requests = %d, want 1 after following the redirect", got) - } - - // A second download is a cache hit. - if again := serveChefRequest(h, proxyPath(t, version.File)); again.Code != http.StatusOK || again.Body.String() != chefTestTarball { - t.Fatalf("cached download = %d %q", again.Code, again.Body.String()) - } - if got := s.count("/api/v1/cookbooks/apt/versions/7.5.22/download"); got != 1 { - t.Errorf("upstream download requests = %d, want 1 (second served from cache)", got) - } - - // With the upstream down and the metadata TTL lapsed, the cached - // cookbook and the stored universe are still served. - s.available.Store(false) - proxy.MetadataTTL = 0 - if offline := serveChefRequest(h, proxyPath(t, version.File)); offline.Code != http.StatusOK || offline.Body.String() != chefTestTarball { - t.Errorf("offline download = %d %q, want the cached tarball", offline.Code, offline.Body.String()) - } - offlineUniverse := serveChefRequest(h, "/universe") - if offlineUniverse.Code != http.StatusOK { - t.Fatalf("offline universe status = %d: %s", offlineUniverse.Code, offlineUniverse.Body.String()) - } - if got := decodeChefUniverse(t, offlineUniverse.Body.Bytes())["apt"]["7.5.22"].DownloadURL; got != version.File { - t.Errorf("offline universe download_url = %q, want %q", got, version.File) - } -} - -func TestChefUpstreamAuthentication(t *testing.T) { - const token = "supermarket-token" - s := newFakeSupermarket(t, token) - // Credentials are scoped to the Supermarket, as upstream.auth scopes - // them by URL, so the storage host it redirects to never sees them. - client := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, - upstreamhttp.AuthFunc(func(requestURL string) (string, string) { - if strings.HasPrefix(requestURL, s.api.URL+"/") { - return "Authorization", "Bearer " + token - } - return "", "" - }))} - h, _ := newChefTestHandler(t, s, client) - - for _, target := range []string{"/universe", "/api/v1/cookbooks/apt", "/api/v1/cookbooks/apt/versions/7.5.22"} { - if w := serveChefRequest(h, target); w.Code != http.StatusOK { - t.Errorf("%s: status = %d, want 200: %s", target, w.Code, w.Body.String()) - } - } - download := serveChefRequest(h, "/api/v1/cookbooks/apt/versions/7.5.22/download") - if download.Code != http.StatusOK || download.Body.String() != chefTestTarball { - t.Fatalf("download = %d %q", download.Code, download.Body.String()) - } - - s.mu.Lock() - defer s.mu.Unlock() - for _, auth := range s.fileAuth { - if auth != "" { - t.Errorf("storage host received Authorization %q", auth) - } - } -} - -func TestChefRejectsInvalidRequests(t *testing.T) { - s := newFakeSupermarket(t, "") - h, _ := newChefTestHandler(t, s, s.api.Client()) - - tests := []struct { - target string - want int - }{ - {"/api/v1/cookbooks/.hidden", http.StatusBadRequest}, - {"/api/v1/cookbooks/apt/versions/latest", http.StatusBadRequest}, - {"/api/v1/cookbooks/apt/versions/1.2.3.4", http.StatusBadRequest}, - {"/api/v1/cookbooks/apt/versions/1.x/download", http.StatusBadRequest}, - {"/api/v1/cookbooks/missing", http.StatusNotFound}, - {"/api/v1/cookbooks/apt/versions/9.9.9", http.StatusNotFound}, - } - for _, tt := range tests { - if w := serveChefRequest(h, tt.target); w.Code != tt.want { - t.Errorf("%s: status = %d, want %d: %s", tt.target, w.Code, tt.want, w.Body.String()) - } - } -} - -// An upstream document that cannot be rewritten is refused rather than -// passed through, which would send clients around the proxy. -func TestChefUnreadableUniverseIsBadGateway(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(`["not","a","universe"]`)) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := NewChefHandler(proxy, chefTestProxyURL, upstream.URL) - - w := serveChefRequest(h, "/universe") - if w.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want 502: %s", w.Code, w.Body.String()) - } - if strings.Contains(w.Body.String(), "not") { - t.Errorf("unreadable document was passed through: %s", w.Body.String()) - } -} - -// Switching upstream.chef to another Supermarket keeps the cache, but an -// archive cached from the previous one is not served for a cookbook the new -// one publishes under the same name and version. -func TestChefArchiveCacheDependsOnUpstream(t *testing.T) { - const target = "/api/v1/cookbooks/apt/versions/7.5.22/download" - public := newFakeSupermarket(t, "") - private := newFakeSupermarket(t, "") - private.tarball = "private cookbook tarball bytes" - - publicHandler, proxy := newChefTestHandler(t, public, public.api.Client()) - if w := serveChefRequest(publicHandler, target); w.Code != http.StatusOK || w.Body.String() != public.tarball { - t.Fatalf("public download = %d %q", w.Code, w.Body.String()) - } - - privateHandler := NewChefHandler(proxy, chefTestProxyURL, private.api.URL) - w := serveChefRequest(privateHandler, target) - if w.Code != http.StatusOK { - t.Fatalf("private download status = %d: %s", w.Code, w.Body.String()) - } - if w.Body.String() != private.tarball { - t.Errorf("private download = %q, want the private Supermarket's tarball", w.Body.String()) - } - if got := private.count(target); got != 1 { - t.Errorf("private download requests = %d, want 1", got) - } - - // Each upstream's archive stays cached under its own identity. - for _, tt := range []struct { - handler *ChefHandler - upstream *fakeSupermarket - }{{publicHandler, public}, {privateHandler, private}} { - if again := serveChefRequest(tt.handler, target); again.Code != http.StatusOK || again.Body.String() != tt.upstream.tarball { - t.Errorf("cached download from %s = %d %q, want %q", tt.upstream.api.URL, again.Code, again.Body.String(), tt.upstream.tarball) - } - if got := tt.upstream.count(target); got != 1 { - t.Errorf("download requests to %s = %d, want 1 (second served from cache)", tt.upstream.api.URL, got) - } - } -} - -func TestChefCacheKeyDependsOnUpstream(t *testing.T) { - a := NewChefHandler(nil, chefTestProxyURL, "https://supermarket.chef.io") - b := NewChefHandler(nil, chefTestProxyURL, "https://supermarket.example.com/") - if a.cacheKey("universe") == b.cacheKey("universe") { - t.Error("two Supermarkets share a universe cache key") - } - if containsPathTraversal(a.cacheKey("cookbooks/apt/versions/7.5.22")) { - t.Error("cache key is not a valid metadata key") - } - if a.archiveFilename("apt", "7.5.22") == b.archiveFilename("apt", "7.5.22") { - t.Error("two Supermarkets share an archive filename") - } - if got := a.archiveFilename("apt", "7.5.22"); !strings.HasPrefix(got, "apt-7.5.22-") || !strings.HasSuffix(got, ".tgz") { - t.Errorf("archive filename = %q, want apt-7.5.22-.tgz", got) - } - if NewChefHandler(nil, chefTestProxyURL, "https://supermarket.example.com").upstreamKey != b.upstreamKey { - t.Error("a trailing slash changes the upstream key") - } -} diff --git a/internal/handler/coalesce_semantics_test.go b/internal/handler/coalesce_semantics_test.go deleted file mode 100644 index 42026f7..0000000 --- a/internal/handler/coalesce_semantics_test.go +++ /dev/null @@ -1,615 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "strings" - "sync" - "testing" - "time" - - "github.com/git-pkgs/artifacts" - "github.com/git-pkgs/registries/fetch" -) - -// runConcurrent runs fn in n goroutines released together and returns their errors. -func runConcurrent(n int, fn func(i int) error) []error { - errs := make([]error, n) - start := make(chan struct{}) - var wg sync.WaitGroup - for i := 0; i < n; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - <-start - errs[i] = fn(i) - }(i) - } - close(start) - wg.Wait() - return errs -} - -// artifactBody builds a one-shot upstream artifact carrying the given bytes. -func artifactBody(content string) *fetch.Artifact { - return &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader(content)), - ContentType: "application/gzip", - } -} - -// drain consumes and closes a CacheResult reader, if there is one. -func drain(res *CacheResult) { - if res != nil && res.Reader != nil { - _, _ = io.Copy(io.Discard, res.Reader) - _ = res.Reader.Close() - } -} - -// TestCoalesceKey_DifferentUpstreamHashDoesNotShare is the safety property that -// makes coalescing sound: callers expecting different bytes must never share a -// fetch, so a re-published version cannot serve stale bytes to a caller that -// asked for the new digest. -func TestCoalesceKey_DifferentUpstreamHashDoesNotShare(t *testing.T) { - const content = "artifact bytes" - proxy, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: content, delay: fetchHoldTime} - proxy.Fetcher = fetcher - - // The digest must carry the "sha256:" prefix; without it the API treats the - // value as unverifiable and clears the hash, which would legitimately let - // the two callers share one fetch. - hashes := []string{ - "sha256:" + sha256Hex(content), - "sha256:" + sha256Hex("something else entirely"), - } - - _ = runConcurrent(2, func(i int) error { - res, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", - "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", hashes[i]) - drain(res) - return err - }) - - if got := fetcher.calls.Load(); got != 2 { - t.Errorf("upstream fetches = %d, want 2: callers expecting different digests must not share a fetch", got) - } -} - -// TestCoalesceKey_HashCasingSharesOneFetch is the other half of that property. -// artifactHashMatches compares digests case-insensitively, so one digest in two -// casings describes one artifact and must not split into two fetches. -func TestCoalesceKey_HashCasingSharesOneFetch(t *testing.T) { - const content = "artifact bytes" - proxy, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: content, delay: fetchHoldTime} - proxy.Fetcher = fetcher - - hex := sha256Hex(content) - digests := []string{"sha256:" + hex, "sha256:" + strings.ToUpper(hex)} - - for i, err := range runConcurrent(2, func(i int) error { - res, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", - "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", digests[i]) - drain(res) - return err - }) { - if err != nil { - t.Fatalf("caller %d failed: %v", i, err) - } - } - - if got := fetcher.calls.Load(); got != 1 { - t.Errorf("upstream fetches = %d, want 1: one digest in two casings is one artifact", got) - } -} - -// TestCoalesceKey_DifferentDownloadURLDoesNotShare covers the other half of the -// key: same package, different upstream URL, must not collapse into one fetch. -func TestCoalesceKey_DifferentDownloadURLDoesNotShare(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: "artifact bytes", delay: fetchHoldTime} - proxy.Fetcher = fetcher - - urls := []string{ - "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", - "https://mirror.example.com/pkg/-/pkg-1.0.0.tgz", - } - - _ = runConcurrent(2, func(i int) error { - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", urls[i]) - drain(res) - return err - }) - - if got := fetcher.calls.Load(); got != 2 { - t.Errorf("upstream fetches = %d, want 2: different upstream URLs must not share a fetch", got) - } -} - -// TestCoalesceKey_DistinctArtifactsDoNotSerialize guards against an over-broad -// key: four packages fetched at once must still produce four fetches. -func TestCoalesceKey_DistinctArtifactsDoNotSerialize(t *testing.T) { - const n = 4 - proxy, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: "artifact bytes", delay: fetchHoldTime} - proxy.Fetcher = fetcher - - names := []string{"alpha", "beta", "gamma", "delta"} - errs := runConcurrent(n, func(i int) error { - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", names[i], "1.0.0", names[i]+"-1.0.0.tgz", - "https://registry.npmjs.org/"+names[i]+"/-/"+names[i]+"-1.0.0.tgz") - drain(res) - return err - }) - for i, err := range errs { - if err != nil { - t.Errorf("caller %d (%s): %v", i, names[i], err) - } - } - if got := fetcher.calls.Load(); got != n { - t.Errorf("upstream fetches = %d, want %d: distinct artifacts must not share a fetch", got, n) - } -} - -// TestCoalesce_FailedFetchReachesEveryCallerAndIsRetriable verifies both claims -// in coalesceFetch's doc comment: a failed fetch reaches every caller sharing -// it, and the key is released so a later request retries. -func TestCoalesce_FailedFetchReachesEveryCallerAndIsRetriable(t *testing.T) { - const callers = 8 - proxy, _, _, fetcher := setupTestProxy(t) - boom := errors.New("upstream unavailable") - fetcher.fetchErr = boom - - errs := runConcurrent(callers, func(int) error { - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", - "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") - drain(res) - return err - }) - for i, err := range errs { - if err == nil { - t.Errorf("caller %d: got nil error, want the shared fetch's failure", i) - } else if !errors.Is(err, boom) { - t.Errorf("caller %d: got %v, want it to wrap %v", i, err, boom) - } - } - - // The key must be released: a later request retries rather than inheriting - // the failure. - fetcher.fetchErr = nil - fetcher.artifact = artifactBody("recovered bytes") - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", - "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") - if err != nil { - t.Fatalf("retry after failed coalesced fetch: %v", err) - } - body, _ := io.ReadAll(res.Reader) - _ = res.Reader.Close() - if string(body) != "recovered bytes" { - t.Errorf("retry body = %q, want %q", body, "recovered bytes") - } -} - -// TestCoalesce_ResolverPath covers the other entry point: GetOrFetchArtifact -// resolves the URL itself, so it is keyed without one. -func TestCoalesce_ResolverPath(t *testing.T) { - const callers = 8 - proxy, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: "resolved artifact bytes", delay: fetchHoldTime} - proxy.Fetcher = fetcher - - errs := runConcurrent(callers, func(int) error { - res, err := proxy.GetOrFetchArtifact(context.Background(), - "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz") - drain(res) - return err - }) - for i, err := range errs { - if err != nil { - t.Errorf("caller %d: %v", i, err) - } - } - if got := fetcher.calls.Load(); got != 1 { - t.Errorf("upstream fetches = %d, want 1", got) - } -} - -// TestCoalesce_ResolverPathEmptyFilename exercises that path when the filename -// is left to be resolved, which the key cannot know up front. -func TestCoalesce_ResolverPathEmptyFilename(t *testing.T) { - const callers = 8 - proxy, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: "resolved artifact bytes", delay: fetchHoldTime} - proxy.Fetcher = fetcher - - errs := runConcurrent(callers, func(int) error { - res, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "left-pad", "1.3.0", "") - drain(res) - return err - }) - for i, err := range errs { - if err != nil { - t.Errorf("caller %d: %v", i, err) - } - } - if got := fetcher.calls.Load(); got != 1 { - t.Errorf("upstream fetches = %d, want 1", got) - } -} - -// TestCoalesce_SubsequentRequestIsACacheHit confirms the coalesced fetch was -// committed and is visible later, not just streamed to the waiting callers. -func TestCoalesce_SubsequentRequestIsACacheHit(t *testing.T) { - const callers = 8 - const url = "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz" - proxy, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: "artifact bytes", delay: fetchHoldTime} - proxy.Fetcher = fetcher - - _ = runConcurrent(callers, func(int) error { - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) - drain(res) - return err - }) - - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) - if err != nil { - t.Fatalf("follow-up request: %v", err) - } - defer func() { _ = res.Reader.Close() }() - if !res.Cached { - t.Error("follow-up request should be served from cache") - } - if got := fetcher.calls.Load(); got != 1 { - t.Errorf("upstream fetches = %d, want 1 after a follow-up cache hit", got) - } -} - -// TestCoalesce_ReadersAreIndependent guards openStoredArtifact: callers sharing -// a fetch each need their own reader, or one closing early breaks the rest. -func TestCoalesce_ReadersAreIndependent(t *testing.T) { - const callers = 8 - const content = "artifact bytes that every caller must receive intact" - proxy, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: content, delay: fetchHoldTime} - proxy.Fetcher = fetcher - - results := make([]*CacheResult, callers) - errs := runConcurrent(callers, func(i int) error { - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", - "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") - results[i] = res - return err - }) - for i, err := range errs { - if err != nil { - t.Fatalf("caller %d: %v", i, err) - } - } - - // Close the first caller's reader before anyone else has read a byte. - _ = results[0].Reader.Close() - - for i := 1; i < callers; i++ { - body, err := io.ReadAll(results[i].Reader) - _ = results[i].Reader.Close() - if err != nil { - t.Errorf("caller %d read after another caller closed: %v", i, err) - continue - } - if string(body) != content { - t.Errorf("caller %d got %q, want %q", i, body, content) - } - } -} - -// TestCoalesce_CanceledWaiterDoesNotWaitForTheSharedFetch checks that joining a -// coalesced fetch does not cost a caller its own cancellation. Without the -// leader/waiter split a waiter is pinned until the shared fetch resolves, -// bounded only by the artifact client timeout, so clients that have already -// gone away keep handler goroutines alive for minutes. -func TestCoalesce_CanceledWaiterDoesNotWaitForTheSharedFetch(t *testing.T) { - const leaderFetch = 2 * time.Second - const url = "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz" - - proxy, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: "artifact bytes", delay: leaderFetch, entered: make(chan struct{})} - proxy.Fetcher = fetcher - - leaderDone := make(chan error, 1) - go func() { - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) - drain(res) - leaderDone <- err - }() - - select { - case <-fetcher.entered: // the leader holds the key and is inside its fetch - case <-time.After(5 * time.Second): - t.Fatal("leader never started its fetch") - } - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - start := time.Now() - _, err := proxy.GetOrFetchArtifactFromURL(ctx, "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) - blocked := time.Since(start) - - if !errors.Is(err, context.Canceled) { - t.Errorf("waiter error = %v, want context.Canceled", err) - } - if blocked > leaderFetch/4 { - t.Errorf("canceled waiter blocked %v, want well under %v: it is pinned to the shared fetch", - blocked, leaderFetch/4) - } - - // A waiter leaving must not disturb the fetch the others share. - if err := <-leaderDone; err != nil { - t.Fatalf("leader failed after a waiter canceled: %v", err) - } - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) - if err != nil { - t.Fatalf("follow-up after leader completed: %v", err) - } - defer func() { _ = res.Reader.Close() }() - if !res.Cached { - t.Error("leader's fetch should have been committed to the cache") - } - if got := fetcher.calls.Load(); got != 1 { - t.Errorf("upstream fetches = %d, want 1", got) - } -} - -// inFlightLen reports how many coalesced fetches are currently registered. -func inFlightLen(p *Proxy) int { - p.fetchMu.Lock() - defer p.fetchMu.Unlock() - return len(p.inFlight) -} - -// TestCoalesce_KeyIsReleasedAfterFetch guards the bug this hand-rolled map can -// have that singleflight could not: a key left behind means later callers join -// a finished entry, see its closed done channel, and are served that stale -// result forever, while the map grows without bound. -func TestCoalesce_KeyIsReleasedAfterFetch(t *testing.T) { - const url = "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz" - proxy, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: "artifact bytes", delay: fetchHoldTime} - proxy.Fetcher = fetcher - - _ = runConcurrent(8, func(int) error { - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) - drain(res) - return err - }) - if n := inFlightLen(proxy); n != 0 { - t.Errorf("in-flight entries after a successful fetch = %d, want 0", n) - } - - // A fresh miss for the same key must start a new fetch, not rejoin the old - // entry. Clearing the cache record forces the miss path again. - if err := proxy.ClearCachedArtifact("npm", "pkg", "1.0.0", "pkg-1.0.0.tgz"); err != nil { - t.Fatalf("clear cached artifact: %v", err) - } - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) - if err != nil { - t.Fatalf("second miss for the same key: %v", err) - } - drain(res) - if got := fetcher.calls.Load(); got != 2 { - t.Errorf("upstream fetches = %d, want 2: the second miss must not reuse the finished entry", got) - } - if n := inFlightLen(proxy); n != 0 { - t.Errorf("in-flight entries at end = %d, want 0", n) - } -} - -// missingFromCache is a recheck that always reports a miss, so the shared fetch -// runs. -func missingFromCache() (artifacts.Artifact, string, bool) { - return artifacts.Artifact{}, "", false -} - -// TestCoalesce_LeaderRechecksCacheBeforeFetching covers the window between a -// caller's own cache lookup and it becoming the leader: a concurrent fetch can -// commit the artifact in that gap, and the leader must serve that rather than -// fetch it a second time. -func TestCoalesce_LeaderRechecksCacheBeforeFetching(t *testing.T) { - const content = "artifact bytes" - proxy, _, store, _ := setupTestProxy(t) - - const storagePath = "npm/pkg/1.0.0/pkg-1.0.0.tgz" - if _, _, err := store.Store(context.Background(), storagePath, strings.NewReader(content)); err != nil { - t.Fatalf("seeding storage: %v", err) - } - committed := artifacts.Artifact{ - PURL: "pkg:npm/pkg@1.0.0", - Filename: "pkg-1.0.0.tgz", - Size: int64(len(content)), - } - - res, err := proxy.coalesceFetch(context.Background(), "any-key", - func() (artifacts.Artifact, string, bool) { return committed, storagePath, true }, - func(context.Context) (artifacts.Artifact, string, error) { - t.Error("fetched an artifact that was already in the cache") - return artifacts.Artifact{}, "", errors.New("commit must not run") - }) - if err != nil { - t.Fatalf("coalesceFetch failed: %v", err) - } - defer drain(res) - got, err := io.ReadAll(res.Reader) - if err != nil { - t.Fatalf("reading result: %v", err) - } - if string(got) != content { - t.Errorf("got %q, want %q", got, content) - } - if n := inFlightLen(proxy); n != 0 { - t.Errorf("in-flight entries = %d, want 0", n) - } -} - -// TestCachedArtifactRecord covers the recheck itself: it must report the row a -// concurrent fetch committed, match its digest the way artifactHashMatches -// does, and report a miss for anything else. -func TestCachedArtifactRecord(t *testing.T) { - const ( - content = "artifact bytes" - pkgPURL = "pkg:npm/pkg" - versionPURL = "pkg:npm/pkg@1.0.0" - filename = "pkg-1.0.0.tgz" - storagePath = "npm/pkg/1.0.0/pkg-1.0.0.tgz" - ) - proxy, _, _, _ := setupTestProxy(t) - - hex := sha256Hex(content) - committed := testArtifact(content, versionPURL, filename, "application/gzip") - if err := proxy.updateCacheDB("npm", "pkg", pkgPURL, - "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", storagePath, committed, ""); err != nil { - t.Fatalf("seeding cache record: %v", err) - } - - for _, tc := range []struct { - name, filename, hash string - want bool - }{ - {"no upstream hash", filename, "", true}, - {"matching hash", filename, hex, true}, - {"matching hash in upper case", filename, strings.ToUpper(hex), true}, - {"different hash", filename, sha256Hex("something else entirely"), false}, - {"unknown filename", "pkg-1.0.0.zip", hex, false}, - } { - t.Run(tc.name, func(t *testing.T) { - got, path, ok := proxy.cachedArtifactRecord(pkgPURL, versionPURL, tc.filename, tc.hash) - if ok != tc.want { - t.Fatalf("ok = %v, want %v", ok, tc.want) - } - if !ok { - return - } - if path != storagePath { - t.Errorf("storage path = %q, want %q", path, storagePath) - } - if got.Digest.Encoded() != hex { - t.Errorf("digest = %q, want %q", got.Digest.Encoded(), hex) - } - }) - } -} - -// TestCoalesce_LeaderFetchesWhenRecheckedBytesAreGone covers the other branch -// of the recheck: a record whose bytes no longer open is not served, and the -// shared fetch runs instead, the same recovery the cache lookup makes. -func TestCoalesce_LeaderFetchesWhenRecheckedBytesAreGone(t *testing.T) { - const content = "fetched bytes" - const storagePath = "npm/pkg/1.0.0/pkg-1.0.0.tgz" - proxy, _, store, _ := setupTestProxy(t) - - stale := artifacts.Artifact{PURL: "pkg:npm/pkg@1.0.0", Filename: "pkg-1.0.0.tgz"} - if _, err := store.Open(context.Background(), storagePath); err == nil { - t.Fatal("stale bytes were present, so the test proves nothing") - } - - // The leader runs commit on its own goroutine, so a plain counter is safe. - fetches := 0 - res, err := proxy.coalesceFetch(context.Background(), "any-key", - func() (artifacts.Artifact, string, bool) { return stale, storagePath, true }, - func(ctx context.Context) (artifacts.Artifact, string, error) { - fetches++ - if _, _, err := store.Store(ctx, storagePath, strings.NewReader(content)); err != nil { - return artifacts.Artifact{}, "", err - } - return testArtifact(content, stale.PURL, stale.Filename, "application/gzip"), storagePath, nil - }) - if err != nil { - t.Fatalf("coalesceFetch failed: %v", err) - } - defer drain(res) - if fetches != 1 { - t.Errorf("shared fetches = %d, want 1: a record without bytes must be refetched", fetches) - } - got, err := io.ReadAll(res.Reader) - if err != nil { - t.Fatalf("reading result: %v", err) - } - if string(got) != content { - t.Errorf("got %q, want %q", got, content) - } - if n := inFlightLen(proxy); n != 0 { - t.Errorf("in-flight entries = %d, want 0", n) - } -} - -// TestCoalesce_PanicInSharedFetchDoesNotStrandWaiters checks the failure mode -// that matters most: a caller parked on a shared fetch must never be left -// blocked forever when that fetch dies. -// -// This drives coalesceFetch directly and holds the shared entry itself, because -// whether a second caller has reached the wait is not observable from outside: -// it runs a cache lookup against the database first, so releasing the leader on -// a timer races that query. Losing the race made a second caller the leader -// instead of a waiter, and its panic was unrecovered, killing the test binary -// rather than failing the test. -func TestCoalesce_PanicInSharedFetchDoesNotStrandWaiters(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - const key = "pkg:npm/pkg@1.0.0\x00pkg-1.0.0.tgz" - - inCommit := make(chan struct{}) - release := make(chan struct{}) - leaderPanicked := make(chan struct{}) - - go func() { - defer func() { - _ = recover() // the panic surfaces in the leader, as it would in a handler - close(leaderPanicked) - }() - _, _ = proxy.coalesceFetch(context.Background(), key, missingFromCache, - func(context.Context) (artifacts.Artifact, string, error) { - close(inCommit) - <-release - panic("upstream fetch exploded") - }) - }() - - <-inCommit // the leader holds the key and is inside the fetch - - // Take the entry a waiter would park on, while the leader is still held. - proxy.fetchMu.Lock() - shared := proxy.inFlight[key] - proxy.fetchMu.Unlock() - if shared == nil { - t.Fatal("no in-flight entry registered for a running fetch") - } - - close(release) - - select { - case <-shared.done: - case <-time.After(5 * time.Second): - t.Fatal("waiter stranded: a panicking shared fetch never released its waiters") - } - if !errors.Is(shared.err, errSharedFetchAbandoned) { - t.Errorf("waiter error = %v, want errSharedFetchAbandoned", shared.err) - } - - <-leaderPanicked - if n := inFlightLen(proxy); n != 0 { - t.Errorf("in-flight entries after a panic = %d, want 0", n) - } -} diff --git a/internal/handler/coalesce_test.go b/internal/handler/coalesce_test.go deleted file mode 100644 index baf052d..0000000 --- a/internal/handler/coalesce_test.go +++ /dev/null @@ -1,185 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "io" - "log/slog" - "net/http" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/storage" - "github.com/git-pkgs/registries/fetch" -) - -// fetchHoldTime holds each stub fetch open long enough that concurrent callers -// reliably overlap inside it. The exact value is not significant. -const fetchHoldTime = 50 * time.Millisecond - -// countingFetcher counts upstream fetches and holds each one open. -type countingFetcher struct { - calls atomic.Int64 - content string - delay time.Duration - - // entered, if set, is closed when the first fetch begins. A test can wait - // on it to know the leader holds the key, rather than guessing with a - // sleep. - entered chan struct{} - enterOnce sync.Once -} - -func (f *countingFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { - return f.FetchWithHeaders(ctx, url, nil) -} - -func (f *countingFetcher) FetchWithHeaders(_ context.Context, _ string, _ http.Header) (*fetch.Artifact, error) { - f.calls.Add(1) - if f.entered != nil { - f.enterOnce.Do(func() { close(f.entered) }) - } - time.Sleep(f.delay) - return &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader(f.content)), - ContentType: "application/gzip", - }, nil -} - -func (f *countingFetcher) Head(context.Context, string) (int64, string, error) { - return 0, "", nil -} - -// TestGetOrFetchArtifactFromURL_ConcurrentMissesCoalesce asserts that N -// simultaneous misses for one artifact produce a single upstream fetch. That is -// the CI shape: parallel jobs installing overlapping dependencies cold. -func TestGetOrFetchArtifactFromURL_ConcurrentMissesCoalesce(t *testing.T) { - const goroutines = 8 - const content = "left-pad tarball bytes" - - proxy, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: content, delay: fetchHoldTime} - proxy.Fetcher = fetcher - - start := make(chan struct{}) - var wg sync.WaitGroup - errs := make([]error, goroutines) - bodies := make([]string, goroutines) - - for i := 0; i < goroutines; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - <-start - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", - "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz") - if err != nil { - errs[i] = err - return - } - defer func() { _ = res.Reader.Close() }() - b, err := io.ReadAll(res.Reader) - errs[i] = err - bodies[i] = string(b) - }(i) - } - - close(start) - wg.Wait() - - for i, err := range errs { - if err != nil { - t.Errorf("goroutine %d: unexpected error: %v", i, err) - } - } - // Every caller must get its own intact copy of the bytes. - for i, b := range bodies { - if b != content { - t.Errorf("goroutine %d: body = %q, want %q", i, b, content) - } - } - if got := fetcher.calls.Load(); got != 1 { - t.Errorf("upstream fetches = %d, want 1 (%d concurrent callers stampeded the upstream)", got, goroutines) - } -} - -// TestGetOrFetchArtifactFromURL_ConcurrentMissesFileStorage runs the same -// scenario against the real file:// backend, the default in production. -// -// Uncoalesced this fails outright, not merely wastefully. Every caller stores -// to one key, and fileblob rewrites a ".attrs" sidecar per key with os.Create, -// truncating in place outside the rename that protects the blob. Decoding that -// sidecar mid-truncate gives "opening reader: EOF", served as a 502. -// -// Only the fetcher is stubbed, because the real one refuses loopback so an -// httptest upstream is unreachable. The storage, where this fails, is real. -func TestGetOrFetchArtifactFromURL_ConcurrentMissesFileStorage(t *testing.T) { - const goroutines = 16 - content := bytes.Repeat([]byte("tarball-bytes-"), 512) - - ctx := context.Background() - dir := t.TempDir() - - db, err := database.Create(filepath.Join(dir, "test.db")) - if err != nil { - t.Fatalf("create database: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - - store, err := storage.OpenBucket(ctx, "file://"+filepath.Join(dir, "cache")) - if err != nil { - t.Fatalf("open storage: %v", err) - } - t.Cleanup(func() { _ = store.Close() }) - - fetcher := &countingFetcher{content: string(content), delay: fetchHoldTime} - proxy := NewProxy(db, store, fetcher, fetch.NewResolver(), - slog.New(slog.NewTextHandler(io.Discard, nil))) - - start := make(chan struct{}) - var wg sync.WaitGroup - errs := make([]error, goroutines) - bodies := make([][]byte, goroutines) - - for i := 0; i < goroutines; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - <-start - res, err := proxy.GetOrFetchArtifactFromURL(ctx, - "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", - "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz") - if err != nil { - errs[i] = err - return - } - defer func() { _ = res.Reader.Close() }() - body, readErr := io.ReadAll(res.Reader) - errs[i] = readErr - bodies[i] = body - }(i) - } - - close(start) - wg.Wait() - - for i, err := range errs { - if err != nil { - t.Errorf("caller %d failed: %v", i, err) - } - } - for i, body := range bodies { - if !bytes.Equal(body, content) { - t.Errorf("caller %d got %d bytes, want %d", i, len(body), len(content)) - } - } - if got := fetcher.calls.Load(); got != 1 { - t.Errorf("upstream fetches = %d, want 1", got) - } -} diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 6b82cff..bc3bc1d 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -1,61 +1,32 @@ package handler import ( - "bytes" "context" "encoding/json" "errors" "fmt" + "io" "net/http" - "net/url" "path" - "regexp" "strings" "time" - "github.com/git-pkgs/proxy/internal/jsonscan" - "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" ) const ( composerUpstream = "https://packagist.org" composerRepo = "https://repo.packagist.org" - composerMinified = "composer/2.0" composerUnset = "__unset" - composerDevReset = "~dev" - composerDevFile = "~dev" // suffix of the metadata file holding dev versions vendorPackageParts = 2 ) -var composerUnsetJSON = []byte(`"` + composerUnset + `"`) - -// Archives are stored under pkg:composer/{vendor}/{package}, which purl -// lowercases, so a package rule matches whatever case it is written in. The -// spec also claims the "packagist" spelling git-pkgs writes into -// packages.ecosystem. -func init() { - retention.Default.Register(retention.Spec{Key: "composer", CanonicalPackage: composerRetentionKey}) -} - -// composerRetentionKey maps a package rule to the stored package PURL. A -// key without a vendor parses fine but can never match, since every stored -// package has one, so it is refused. -func composerRetentionKey(p *purl.PURL) (string, bool) { - if p == nil || p.Namespace == "" { - return "", false - } - return retention.DefaultCanonical("composer")(p) -} - // ComposerHandler handles Composer/Packagist registry protocol requests. type ComposerHandler struct { proxy *Proxy upstreamURL string repoURL string proxyURL string - routes packageRoutes - indexCache composerIndexCache } // NewComposerHandler creates a new Composer protocol handler. @@ -68,22 +39,6 @@ func NewComposerHandler(proxy *Proxy, proxyURL string) *ComposerHandler { } } -// NewComposerHandlerWithUpstreams creates a Composer handler with custom API -// and repository upstreams. -func NewComposerHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, repoURL string) *ComposerHandler { - h := NewComposerHandler(proxy, proxyURL) - h.upstreamURL = configuredUpstreamURL(upstreamURL, composerUpstream) - h.repoURL = configuredUpstreamURL(repoURL, composerRepo) - return h -} - -// WithPackageRoutes sends packages matching a pattern to a dedicated Composer -// repository instead of the default one. See packageRoutes. -func (h *ComposerHandler) WithPackageRoutes(routes map[string]string) *ComposerHandler { - h.routes = newPackageRoutes(routes) - return h -} - // Routes returns the HTTP handler for Composer requests. func (h *ComposerHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -101,20 +56,12 @@ func (h *ComposerHandler) Routes() http.Handler { mux.HandleFunc("GET /search.json", h.proxyUpstream) mux.HandleFunc("GET /packages/list.json", h.proxyUpstream) - // Security advisories and filter lists, as the default repository offers them - mux.HandleFunc("GET "+composerAdvisoriesPath, h.handleSecurityAdvisories) - mux.HandleFunc("POST "+composerAdvisoriesPath, h.handleSecurityAdvisories) - mux.HandleFunc("GET "+composerFilterSummaryPath, h.handleFilterSummary) - return mux } -// handleServiceIndex returns the Composer repository service index. It -// carries the security advisories and filter list entries of the default -// repository's own index when that has them, kept in memory for the metadata -// TTL; if the repository has not been reached, the index goes out without -// them. +// handleServiceIndex returns the Composer repository service index. func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request) { + // Return a minimal service index pointing to our proxy index := map[string]any{ "packages": map[string]any{}, "metadata-url": h.proxyURL + "/composer/p2/%package%.json", @@ -122,26 +69,11 @@ func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Requ "search": h.proxyURL + "/composer/search.json?q=%query%&type=%type%", "providers-lazy-url": h.proxyURL + "/composer/p2/%package%.json", } - if upstream, ok := h.upstreamIndex(r.Context()); ok { - upstream.advertise(index, h.proxyURL) - } - w.Header().Set(headerContentType, "application/json") + w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(index) } -// sourceFor returns the repository serving a package and the key its metadata -// is cached under. packageName may carry the ~dev suffix of the file holding -// the package's dev versions: routes match the package without it, so its dev -// versions come from the same repository, while the cache key keeps it so the -// two files stay apart. -func (h *ComposerHandler) sourceFor(packageName string) (repoURL, cacheKey string) { - if route, routed := h.routes.match(strings.TrimSuffix(packageName, composerDevFile)); routed { - return route.url, route.cacheKey(packageName) - } - return h.repoURL, packageName -} - // handlePackageMetadata proxies and rewrites package metadata. func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { // Parse path: /p2/{vendor}/{package}.json @@ -156,257 +88,173 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R pkg := parts[1] packageName := vendor + "/" + pkg - h.proxy.Logger.Debug("composer metadata request", "package", packageName) + h.proxy.Logger.Info("composer metadata request", "package", packageName) - repoURL, cacheKey := h.sourceFor(packageName) - upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg) + upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg) - basePackage := strings.TrimSuffix(packageName, composerDevFile) - local, err := h.composerLocalEntries(basePackage, basePackage != packageName) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "composer", packageName, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to read pushed composer versions", "package", basePackage, "error", err) - http.Error(w, "failed to read pushed versions", http.StatusInternalServerError) - return - } - if h.proxy.hostedOnly("composer", basePackage) { - h.serveHostedOnlyMetadata(w, r, basePackage, cacheKey, local) - return - } - if len(local) > 0 { - h.serveMetadataWithLocal(w, r, basePackage, upstreamURL, cacheKey, local) - return - } - - if stored, ok := h.proxy.storedRewrite("composer", cacheKey, h.proxyURL, cacheKey); ok { - serveRewrittenMetadata(w, r, stored) - return - } - - upstream := h.proxy.fetchMetadataDocument(r.Context(), "composer", cacheKey, upstreamURL, contentTypeJSON, nil) - if err := upstream.err; err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) return } - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, cacheKey, upstream.body, upstream.digest, h.rewriteMetadataKeeping) + rewritten, err := h.rewriteMetadata(body) if err != nil { - if r.Context().Err() != nil { - return // the client left while waiting on a shared rewrite - } - if h.proxy.mustFilterMetadata("composer", basePackage) { - h.proxy.Logger.Error("failed to filter composer metadata", "package", basePackage, "error", err) - http.Error(w, "failed to filter package metadata", http.StatusBadGateway) - return - } h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) - markStale(w, upstream.stale) - w.Header().Set(headerContentType, "application/json") - _, _ = w.Write(upstream.body) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) return } - markStale(w, upstream.stale) - serveRewrittenMetadata(w, r, rewritten) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(rewritten) } -// rewriteMetadataKeeping rewrites dist URLs and filters versions in cooldown. -// It returns the kept package names and versions without expanding minified lists. -func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, error) { - return h.rewriteMetadataWithLocal(body, "", nil) -} - -// rewriteMetadataWithLocal is rewriteMetadataKeeping that also lists the -// pushed versions in local under localPackage, in place of any upstream entry -// with the same version. Cooldown only filters upstream versions. A document -// that does not list localPackage at all is replaced by one holding only the -// pushed versions. -func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage string, local []composerLocalEntry) ([]byte, []string, error) { - if !json.Valid(body) { - return nil, nil, errors.New("composer metadata is not valid JSON") - } - format, _, err := jsonscan.LookupString(body, "minified") - if err != nil { - return nil, nil, err - } - minified := format == composerMinified - - var out bytes.Buffer - var kept []string - listed := false - out.Grow(len(body) + len(body)/8) - err = jsonscan.RewriteMembers(&out, body, func(out *bytes.Buffer, m jsonscan.Member) (bool, error) { - packages := m.Value(body) - if !jsonscan.KeyIs(m.Key(body), "packages") || packages[0] != '{' { - return false, nil - } - return true, jsonscan.RewriteMembers(out, packages, func(out *bytes.Buffer, p jsonscan.Member) (bool, error) { - versions := p.Value(packages) - if versions[0] != '[' { - return false, nil - } - packageName, err := jsonscan.Key(p.Key(packages)) - if err != nil { - return false, err - } - var pushed []composerLocalEntry - if packageName == localPackage { - pushed, listed = local, true - } - return true, h.writeVersions(out, packageName, versions, minified, &kept, pushed) - }) - }) - if err != nil { - return nil, nil, err - } - if len(local) > 0 && !listed { - document, err := localComposerDocument(localPackage, local) - return document, nil, err - } - return out.Bytes(), kept, nil -} - -// writeVersions writes one package's version list without the versions in -// cooldown and with each dist pointing at this proxy, and appends each version -// it writes to kept as its package name and version. -// -// Minified lists stay minified. Each version written carries the fields that -// differ from what the client has expanded so far, which is the upstream -// entry itself unless filtered versions were dropped before it; then it also -// carries the changes those versions made. Every version gets its own dist, -// since the proxied URL names the version. -// -// The pushed versions in local follow the upstream ones and take the place of -// upstream entries with the same version. -func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool, kept *[]string, local []composerLocalEntry) error { - packagePURL := canonicalPackagePURL("composer", packageName) - upstream, written := newComposerFields(), newComposerFields() - devReset, first := false, true - _, routed := h.routes.match(packageName) - - out.WriteByte('[') - err := jsonscan.EachElement(versions, func(entry []byte) error { - if minified && jsonscan.StringIs(entry, composerDevReset) { - upstream.reset() - written.reset() - devReset = true - return nil - } - if entry[0] != '{' { - return nil - } - if !minified { - upstream.reset() - written.reset() - } - if err := upstream.apply(entry, minified); err != nil { - return err - } - - version := upstream.string("version") - if pushedComposerVersion(local, version) || h.shouldFilterVersion(packagePURL, packageName, version, upstream.get("time")) { - return nil - } - - if !first { - out.WriteByte(',') - } - first = false - if devReset { - out.WriteString(`"` + composerDevReset + `",`) - devReset = false - } - if routed { - disableDefaultNotification(upstream) - } - h.writeVersion(out, packageName, version, upstream, written) - *kept = append(*kept, packageName+"\x00"+version) - return nil - }) - if err != nil { - return err - } - for _, e := range local { - pushed := newComposerFields() - if err := pushed.apply(e.entry, false); err != nil { - return err - } - if !minified { - written.reset() - } - if !first { - out.WriteByte(',') - } - first = false - if devReset { - out.WriteString(`"` + composerDevReset + `",`) - devReset = false - } - h.writeVersion(out, packageName, e.version, pushed, written) - *kept = append(*kept, packageName+"\x00"+e.version) - } - out.WriteByte(']') - return nil -} - -// writeVersion writes the fields of upstream that differ from written, with -// dist pointing at this proxy, and "__unset" for those written has and -// upstream lacks. written is then what the client expands this version to. -func (h *ComposerHandler) writeVersion(out *bytes.Buffer, packageName, version string, upstream, written *composerFields) { - out.WriteByte('{') - first := true - writeMember := func(key, value []byte) { - if !first { - out.WriteByte(',') - } - first = false - out.Write(key) - out.WriteByte(':') - out.Write(value) +// rewriteMetadata rewrites dist URLs in Composer metadata to point at this proxy. +// If the metadata uses the minified Composer v2 format, it is expanded first so +// that every version entry contains all fields. If cooldown is enabled, versions +// published too recently are filtered out. +func (h *ComposerHandler) rewriteMetadata(body []byte) ([]byte, error) { + var metadata map[string]any + if err := json.Unmarshal(body, &metadata); err != nil { + return nil, err } - for _, f := range upstream.fields { - value := f.value - if value == nil { + packages, ok := metadata["packages"].(map[string]any) + if !ok { + return body, nil + } + + minified := metadata["minified"] == "composer/2.0" + + for packageName, versions := range packages { + versionList, ok := versions.([]any) + if !ok { continue } - if f.name == "dist" { - value = h.rewriteDist(packageName, version, value) + + if minified { + versionList = expandMinifiedVersions(versionList) } - if bytes.Equal(value, written.get(f.name)) { + + packages[packageName] = h.filterAndRewriteVersions(packageName, versionList) + } + + delete(metadata, "minified") + + return json.Marshal(metadata) +} + +// expandMinifiedVersions expands the Composer v2 minified format where each +// version entry only contains fields that differ from the previous entry. +// The "~dev" sentinel string resets the inheritance chain, and the "__unset" +// value removes a field from the inherited state. +func expandMinifiedVersions(versionList []any) []any { + expanded := make([]any, 0, len(versionList)) + inherited := map[string]any{} + + for _, v := range versionList { + // The "~dev" sentinel resets the inheritance chain for dev versions. + if s, ok := v.(string); ok && s == "~dev" { + inherited = map[string]any{} continue } - writeMember(f.key, value) - written.set(f.name, f.key, value) - } - for _, f := range written.fields { - if f.value != nil && upstream.get(f.name) == nil { - writeMember(f.key, composerUnsetJSON) - written.set(f.name, f.key, nil) + + vmap, ok := v.(map[string]any) + if !ok { + continue } + + // Merge inherited fields into a new map, then overlay current fields. + // Deep copy values to avoid shared references between versions. + merged := make(map[string]any, len(inherited)+len(vmap)) + for k, val := range inherited { + merged[k] = deepCopyValue(val) + } + for k, val := range vmap { + if val == composerUnset { + delete(merged, k) + continue + } + merged[k] = val + } + + // Update inherited state for next iteration. + inherited = merged + + expanded = append(expanded, merged) } - out.WriteByte('}') + + return expanded +} + +// deepCopyValue returns a deep copy of JSON-like values (maps, slices, scalars). +func deepCopyValue(v any) any { + switch val := v.(type) { + case map[string]any: + m := make(map[string]any, len(val)) + for k, v := range val { + m[k] = deepCopyValue(v) + } + return m + case []any: + s := make([]any, len(val)) + for i, v := range val { + s[i] = deepCopyValue(v) + } + return s + default: + return v + } +} + +// filterAndRewriteVersions applies cooldown filtering and rewrites dist URLs +// for a single package's version list. +func (h *ComposerHandler) filterAndRewriteVersions(packageName string, versionList []any) []any { + packagePURL := purl.MakePURLString("composer", packageName, "") + + filtered := versionList[:0] + for _, v := range versionList { + vmap, ok := v.(map[string]any) + if !ok { + continue + } + + version, _ := vmap["version"].(string) + + if h.shouldFilterVersion(packagePURL, packageName, version, vmap) { + continue + } + + h.rewriteDistURL(vmap, packageName, version) + filtered = append(filtered, v) + } + + return filtered } // shouldFilterVersion returns true if the version should be excluded due to cooldown. -func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version string, published []byte) bool { +func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version string, vmap map[string]any) bool { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { return false } - publishedAt, ok := composerPublishTime(published) + timeStr, ok := vmap["time"].(string) if !ok { return false } - if !h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) { + publishedAt, err := time.Parse(time.RFC3339, timeStr) + if err != nil { + return false + } + + if !h.proxy.Cooldown.IsAllowed("composer", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering composer version", "package", packageName, "version", version) return true @@ -415,215 +263,37 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version return false } -// composerPublishTime parses the raw "time" field of a version, reporting -// false when it is missing or not an RFC 3339 string. -func composerPublishTime(published []byte) (time.Time, bool) { - var timeStr string - if err := json.Unmarshal(published, &timeStr); err != nil { - return time.Time{}, false - } - publishedAt, err := time.Parse(time.RFC3339, timeStr) - return publishedAt, err == nil -} - -// disableDefaultNotification stops Composer from reporting installs of a routed -// package to the default repository's notify-batch URL, which would disclose -// the package name to it. Composer only falls back to notify-batch when a -// version has no notification-url of its own, and skips an empty one. Setting -// it on the expanded fields keeps it set for the versions that inherit them. -// Composer reads a null notification-url as absent too. -func disableDefaultNotification(fields *composerFields) { - if v := fields.get("notification-url"); v == nil || string(v) == "null" { - fields.set("notification-url", []byte(`"notification-url"`), []byte(`""`)) - } -} - -// rewriteDist returns a version's dist object with its url pointing at this -// proxy. A dist without a string url is returned unchanged. -func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte { - urlMember, ok, err := jsonscan.FindMember(dist, "url") - if err != nil || !ok || dist[urlMember.ValStart] != '"' { - return dist - } - var upstreamURL string - if err := json.Unmarshal(urlMember.Value(dist), &upstreamURL); err != nil { - return dist - } - distType, _, _ := jsonscan.LookupString(dist, "type") - - proxyURL, ok := h.proxyDistURL(packageName, version, upstreamURL, distType) +// rewriteDistURL rewrites the dist URL in a version entry to point at this proxy. +func (h *ComposerHandler) rewriteDistURL(vmap map[string]any, packageName, version string) { + dist, ok := vmap["dist"].(map[string]any) if !ok { - return dist + return } - encoded, err := json.Marshal(proxyURL) - if err != nil { - return dist - } - rewritten := make([]byte, 0, len(dist)-len(urlMember.Value(dist))+len(encoded)) - rewritten = append(rewritten, dist[:urlMember.ValStart]...) - rewritten = append(rewritten, encoded...) - return append(rewritten, dist[urlMember.ValEnd:]...) -} -// proxyDistURL returns the URL this proxy serves a version's archive at, or -// false when the upstream URL or package name gives nothing to build it from. -func (h *ComposerHandler) proxyDistURL(packageName, version, upstreamURL, distType string) (string, bool) { - if upstreamURL == "" { - return "", false + url, ok := dist["url"].(string) + if !ok || url == "" { + return } - parts := strings.SplitN(packageName, "/", vendorPackageParts) - if len(parts) != vendorPackageParts { - return "", false - } - return fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", - h.proxyURL, parts[0], parts[1], url.PathEscape(version), distFilename(upstreamURL, distType)), true -} -// distFilename returns the file name this proxy serves an archive under: the -// last segment of its upstream URL. -func distFilename(upstreamURL, distType string) string { filename := "package.zip" - if idx := strings.LastIndex(upstreamURL, "/"); idx >= 0 { - filename = upstreamURL[idx+1:] + if idx := strings.LastIndex(url, "/"); idx >= 0 { + filename = url[idx+1:] } // GitHub zipball URLs end with a bare commit hash (no extension). // Append .zip so the archives library can detect the format. - if path.Ext(filename) == "" && distType == "zip" { - filename += ".zip" - } - return filename -} - -// composerCommitPattern matches a full git commit hash, SHA-1 or SHA-256. -var composerCommitPattern = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`) - -// upstreamDistURL returns the upstream URL of the archive served as filename, -// given the dist URL the version's metadata lists now. A branch version only -// lists its current head while lock files pin older commits, so for a branch -// a dist URL ending in a commit hash is rebuilt for the requested commit. A -// tagged release has to ask for the listed name. Any other name gives false: -// fetching the listed archive would cache it under a name it does not have. -func upstreamDistURL(listedURL, distType, filename string, branch bool) (string, bool) { - listed := distFilename(listedURL, distType) - // Proxy versions before the .zip suffix handed out bare commit names, - // and lock files written back then still ask for them. - if bare := distFilename(listedURL, ""); listed != bare && path.Ext(filename) == "" { - filename += listed[len(bare):] - } - if filename == listed { - return listedURL, true - } - if commit, ok := shaQueryCommit(listed); ok { - requested, ok := shaQueryCommit(filename) - listedSha, requestedSha := "sha="+commit, "sha="+requested - idx := strings.LastIndex(listed, listedSha) - if !branch || !ok || idx < 0 || listed[:idx]+requestedSha+listed[idx+len(listedSha):] != filename { - return "", false + if path.Ext(filename) == "" { + if distType, _ := dist["type"].(string); distType == "zip" { + filename += ".zip" } - idx = strings.LastIndex(listedURL, listedSha) - return listedURL[:idx] + requestedSha + listedURL[idx+len(listedSha):], true } - ext := path.Ext(listed) - commit := strings.TrimSuffix(listed, ext) - requested, ok := strings.CutSuffix(filename, ext) - if !branch || !ok || !composerCommitPattern.MatchString(commit) || !composerCommitPattern.MatchString(requested) { - return "", false + + parts := strings.SplitN(packageName, "/", vendorPackageParts) + if len(parts) == vendorPackageParts { + newURL := fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", + h.proxyURL, parts[0], parts[1], version, filename) + dist["url"] = newURL } - idx := strings.LastIndex(listedURL, commit) - return listedURL[:idx] + requested + listedURL[idx+len(commit):], true -} - -// shaQueryCommit returns the commit an archive name passes in a sha query -// parameter. GitLab names the archive of every commit archive.zip and picks -// the commit that way. -func shaQueryCommit(filename string) (string, bool) { - _, query, ok := strings.Cut(filename, "?") - if !ok { - return "", false - } - values, err := url.ParseQuery(query) - if err != nil { - return "", false - } - commit := values.Get("sha") - return commit, composerCommitPattern.MatchString(commit) -} - -// composerFields holds one version's fields as Composer sees them after -// expanding the minified format: raw values under their decoded names, in the -// order the names first appeared. A removed field keeps its slot with a nil -// value, so the order stays stable when it comes back. -type composerFields struct { - fields []composerField - index map[string]int -} - -type composerField struct { - name string - key, value []byte -} - -func newComposerFields() *composerFields { - return &composerFields{index: map[string]int{}} -} - -func (f *composerFields) reset() { - f.fields = f.fields[:0] - clear(f.index) -} - -// get returns the raw value of the named field, or nil when it is not set. -func (f *composerFields) get(name string) []byte { - if i, ok := f.index[name]; ok { - return f.fields[i].value - } - return nil -} - -// string returns the named field when it is a string, or "". -func (f *composerFields) string(name string) string { - raw := f.get(name) - if len(raw) == 0 || raw[0] != '"' { - return "" - } - var s string - _ = json.Unmarshal(raw, &s) - return s -} - -// set stores value under name, written with the quoted key. A nil value -// removes the field. -func (f *composerFields) set(name string, key, value []byte) { - if i, ok := f.index[name]; ok { - f.fields[i].key, f.fields[i].value = key, value - return - } - f.index[name] = len(f.fields) - f.fields = append(f.fields, composerField{name: name, key: key, value: value}) -} - -// apply overlays the members of a version entry. In minified metadata the -// value "__unset" removes the field instead. -func (f *composerFields) apply(entry []byte, minified bool) error { - return jsonscan.EachMember(entry, func(m jsonscan.Member) error { - key, value := m.Key(entry), m.Value(entry) - if minified && jsonscan.StringIs(value, composerUnset) { - value = nil - } - if bytes.IndexByte(key, '\\') < 0 { - if i, ok := f.index[string(key[1:len(key)-1])]; ok { - f.fields[i].key, f.fields[i].value = key, value - return nil - } - } - name, err := jsonscan.Key(key) - if err != nil { - return err - } - f.set(name, key, value) - return nil - }) } // handleDownload serves a package file, fetching and caching from upstream if needed. @@ -635,44 +305,39 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) packageName := vendor + "/" + pkg - h.proxy.Logger.Debug("composer download request", + h.proxy.Logger.Info("composer download request", "package", packageName, "version", version, "filename", filename) - local, err := h.proxy.GetLocalArtifact(r.Context(), "composer", packageName, version, filename) - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to read pushed package") - return - } - if local != nil { - ServeArtifactRequest(w, r, local) - return - } - if h.proxy.hostedOnly("composer", packageName) { - http.Error(w, "version not found", http.StatusNotFound) - return - } + // We need to fetch the metadata to get the actual download URL since + // Packagist URLs include a hash. Packagist serves dev versions (e.g. + // "3.x-dev", "dev-master") from a separate "~dev" metadata file, while + // tagged releases live in the regular file. Try the file most likely to + // contain this version first, then fall back to the other so that both + // stable and dev versions resolve correctly. + metaURLs := h.metadataURLsForVersion(vendor, pkg, version) - // The archive is cached under the file name from the path, so fetch the - // one that name stands for. Compare it as sent, the way rewriteDist handed - // it out, query included. - requested := path.Base(r.URL.EscapedPath()) - if r.URL.RawQuery != "" { - requested += "?" + r.URL.RawQuery - } else if h.serveCachedRelease(w, r, packageName, version, filename) { - return - } - files := h.metadataFilesForVersion(vendor, pkg, version) - dist, downloadURL, ok := h.cachedDist(r.Context(), files, packageName, version, requested) - if !ok { - if dist, downloadURL, ok = h.upstreamDist(w, r, files, packageName, version, requested); !ok { + h.proxy.Logger.Debug("resolving download URL", + "package", packageName, "version", version, + "metadata_urls", metaURLs) + + var downloadURL string + for _, metaURL := range metaURLs { + url, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version) + if err != nil { + h.proxy.Logger.Error("failed to fetch metadata", "error", err, "url", metaURL) + http.Error(w, "failed to fetch metadata", http.StatusBadGateway) return } + if url != "" { + downloadURL = url + break + } } - h.recordPublishTime(packageName, version, dist.published) - // Lock files name archive URLs directly, so a version filtered out of the - // metadata still has to be withheld here. - if h.shouldFilterVersion(canonicalPackagePURL("composer", packageName), packageName, version, dist.published) { + if downloadURL == "" { + h.proxy.Logger.Debug("version not found in any metadata source", + "package", packageName, "version", version, + "tried_urls", metaURLs) http.Error(w, "version not found", http.StatusNotFound) return } @@ -681,130 +346,14 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) "package", packageName, "version", version, "download_url", downloadURL) - // Archives picked by ?sha= share one file name, so the commit goes into - // the name they are cached under. Proxy versions before that cached every - // commit under the shared name, which is dropped once a commit is fetched. - var sharedFilename string - if commit, ok := shaQueryCommit(requested); ok { - sharedFilename = h.routes.cacheFilename(packageName, filename) - ext := path.Ext(filename) - filename = strings.TrimSuffix(filename, ext) + "-" + commit + ext - } - cacheFilename := h.routes.cacheFilename(packageName, filename) - if commit := strings.TrimSuffix(filename, path.Ext(filename)); isDevVersion(version) && composerCommitPattern.MatchString(commit) { - h.discardOtherCommit(packageName, version, cacheFilename, commit) - } - result, err := h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "composer", packageName, version, cacheFilename, downloadURL, - composerIntegrity(dist.shasum, version, downloadURL == dist.url)) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch package", http.StatusBadGateway) return } - if sharedFilename != "" && !result.Cached { - if err := h.proxy.ClearCachedArtifact("composer", packageName, version, sharedFilename); err != nil { - h.proxy.Logger.Warn("failed to discard composer archive cached under a shared name", - "package", packageName, "version", version, "filename", sharedFilename, "error", err) - } - } - ServeArtifactRequest(w, r, result) -} - -// releaseTimeStored reports whether the publish time of version is kept in -// the versions table, so a download can be checked against cooldown without -// the metadata. That holds for tagged releases of packages from the default -// repository. A branch is left out: its archive and time change with every -// push. So are routed packages, as for npm: the versions row is keyed by the -// plain PURL, which does not say which repository the version came from, so -// a time read from the default repository would be trusted for a private -// version of the same name, and the other way round. -func (h *ComposerHandler) releaseTimeStored(packageName, version string) bool { - if h.proxy.DB == nil || isDevVersion(version) { - return false - } - _, routed := h.routes.match(packageName) - return !routed -} - -// serveCachedRelease serves a tagged release's archive from the cache -// without looking up its dist in the metadata, and reports whether it wrote a -// response. The archive was only cached under its name after that name was -// checked against the metadata, and the cache verifies it against the digest -// recorded then. Cooldown is checked against the publish time recorded when -// the dist was first resolved; without one, as for an archive cached by an -// older proxy, it returns false and the caller takes the metadata path, which -// records it. Requests with a query string, which pick a commit, always take -// that path. -func (h *ComposerHandler) serveCachedRelease(w http.ResponseWriter, r *http.Request, packageName, version, filename string) bool { - if !h.releaseTimeStored(packageName, version) { - return false - } - if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() { - ver, err := h.proxy.DB.GetVersionByPURL(canonicalVersionPURL("composer", packageName, version)) - if err != nil || ver == nil || !ver.PublishedAt.Valid { - return false - } - if !h.proxy.cooldown().IsAllowed("composer", canonicalPackagePURL("composer", packageName), ver.PublishedAt.Time) { - h.proxy.Logger.Info("cooldown: withholding composer archive", - "package", packageName, "version", version) - http.Error(w, "version not found", http.StatusNotFound) - return true - } - } - cached, err := h.proxy.GetCachedArtifact(r.Context(), "composer", packageName, version, filename) - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to read cached package") - return true - } - if cached == nil { - return false - } - ServeArtifactRequest(w, r, cached) - return true -} - -// recordPublishTime stores the publish time a release's metadata lists, for -// serveCachedRelease. It is stored whether or not cooldown is enabled, so -// enabling it later does not send cached archives back to the metadata. -func (h *ComposerHandler) recordPublishTime(packageName, version string, published []byte) { - if !h.releaseTimeStored(packageName, version) { - return - } - publishedAt, ok := composerPublishTime(published) - if !ok { - return - } - if err := h.proxy.DB.SetVersionPublishedAt(canonicalVersionPURL("composer", packageName, version), - canonicalPackagePURL("composer", packageName), publishedAt); err != nil { - h.proxy.Logger.Warn("failed to store composer publish time", - "package", packageName, "version", version, "error", err) - } -} - -// discardOtherCommit clears a cached branch archive named for commit that was -// fetched from another commit's URL. Proxy versions before the pinned-commit -// fix cached the branch head under whatever commit a lock file asked for. The -// record is only cleared while it still points at the object checked here. -func (h *ComposerHandler) discardOtherCommit(packageName, version, cacheFilename, commit string) { - _, versionPURL, err := packagePURLStrings("composer", packageName, version) - if err != nil { - return - } - record, err := h.proxy.DB.GetArtifact(versionPURL, cacheFilename) - if err != nil { - h.proxy.Logger.Warn("failed to read composer cache record", - "purl", versionPURL, "filename", cacheFilename, "error", err) - return - } - if record == nil || !record.StoragePath.Valid || strings.Contains(record.UpstreamURL, commit) { - return - } - h.proxy.Logger.Warn("cached composer archive is from another commit, discarding", - "purl", versionPURL, "filename", cacheFilename, "upstream_url", record.UpstreamURL) - if err := h.proxy.DB.DiscardArtifact(versionPURL, cacheFilename, record.StoragePath.String); err != nil { - h.proxy.Logger.Warn("failed to discard composer cache record", - "purl", versionPURL, "filename", cacheFilename, "error", err) - } + ServeArtifact(w, result) } // isDevVersion reports whether a Composer version string refers to a @@ -815,252 +364,102 @@ func isDevVersion(version string) bool { return strings.HasPrefix(version, "dev-") || strings.HasSuffix(version, "-dev") } -// composerMetadataFile is one of the metadata documents a version's dist can -// be listed in: its upstream URL and the key the proxy caches it under. -type composerMetadataFile struct { - url, cacheKey string -} - -// metadataFilesForVersion returns the metadata documents to consult for a +// metadataURLsForVersion returns the upstream metadata URLs to consult for a // given version, in priority order. Dev versions are served from the "~dev" // file, tagged releases from the regular file; the other file is included as a -// fallback so an unexpected classification still resolves. The cache keys are -// the ones handlePackageMetadata stores the documents under. -func (h *ComposerHandler) metadataFilesForVersion(vendor, pkg, version string) []composerMetadataFile { - file := func(name string) composerMetadataFile { - repoURL, cacheKey := h.sourceFor(name) - return composerMetadataFile{url: fmt.Sprintf("%s/p2/%s.json", repoURL, name), cacheKey: cacheKey} - } - stable := file(vendor + "/" + pkg) - dev := file(vendor + "/" + pkg + composerDevFile) +// fallback so an unexpected classification still resolves. +func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) []string { + stable := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg) + dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", h.repoURL, vendor, pkg) if isDevVersion(version) { - return []composerMetadataFile{dev, stable} + return []string{dev, stable} } - return []composerMetadataFile{stable, dev} -} - -// cachedDist looks up the dist of version in the metadata documents the proxy -// caches for Composer, returning it with the URL to download the requested -// file from. While the documents are fresh this costs no upstream request, -// and when upstream fails their cached copies still resolve the archive. ok is -// false when neither document lists a dist the requested file stands for, as -// for a version published after the metadata was cached, so the caller asks -// upstream instead. Without cache_metadata there is nothing to look in. -func (h *ComposerHandler) cachedDist(ctx context.Context, files []composerMetadataFile, packageName, version, requested string) (composerDist, string, bool) { - if !h.proxy.CacheMetadata || h.proxy.DB == nil { - return composerDist{}, "", false - } - for _, f := range files { - doc := h.proxy.fetchMetadataDocument(ctx, "composer", f.cacheKey, f.url, contentTypeJSON, nil) - if doc.err != nil { - continue - } - dist, err := composerDistURL(doc.body, packageName, version) - if err != nil || dist.url == "" { - continue - } - if downloadURL, ok := upstreamDistURL(dist.url, dist.distType, requested, isDevVersion(version)); ok { - return dist, downloadURL, true - } - } - return composerDist{}, "", false -} - -// upstreamDist looks up the dist of version in the metadata upstream serves -// now, returning it with the URL to download the requested file from. When it -// cannot, it writes the error response and returns false. -func (h *ComposerHandler) upstreamDist(w http.ResponseWriter, r *http.Request, files []composerMetadataFile, packageName, version, requested string) (composerDist, string, bool) { - h.proxy.Logger.Debug("resolving download URL", - "package", packageName, "version", version) - - var dist composerDist - for _, f := range files { - found, err := h.findDownloadURLFromMetadata(r.Context(), f.url, packageName, version) - if err != nil { - if writeUpstreamHeld(w, err) { - return composerDist{}, "", false - } - h.proxy.Logger.Error("failed to fetch metadata", "error", err, "url", f.url) - http.Error(w, "failed to fetch metadata", http.StatusBadGateway) - return composerDist{}, "", false - } - if found.url != "" { - dist = found - break - } - } - - if dist.url == "" { - h.proxy.Logger.Debug("version not found in any metadata source", - "package", packageName, "version", version) - http.Error(w, "version not found", http.StatusNotFound) - return composerDist{}, "", false - } - - downloadURL, ok := upstreamDistURL(dist.url, dist.distType, requested, isDevVersion(version)) - if !ok { - h.proxy.Logger.Info("composer file not in upstream metadata", - "package", packageName, "version", version, - "filename", requested, "listed_url", dist.url) - http.Error(w, "file not found", http.StatusNotFound) - return composerDist{}, "", false - } - return dist, downloadURL, true -} - -// composerDist is what a version's metadata says about its archive: the dist -// URL, type and shasum, and the raw publish time the cooldown is checked -// against. -type composerDist struct { - url, distType, shasum string - published []byte + return []string{stable, dev} } // findDownloadURLFromMetadata fetches a metadata document and returns the dist -// of the given version, with an empty URL if the version is not present. A -// missing document (404 or 410) or a missing version both yield an empty URL -// so the caller can fall back; any other failure is an error. -func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (composerDist, error) { +// URL for the given version, or an empty string if the version is not present. +// An error is returned only on transport failure; a missing document (non-200) +// or a missing version both yield an empty string so the caller can fall back. +func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (string, error) { h.proxy.Logger.Debug("fetching upstream metadata for download lookup", "url", metaURL, "package", packageName, "version", version) req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaURL, nil) if err != nil { - return composerDist{}, err + return "", err } resp, err := h.proxy.HTTPClient.Do(req) if err != nil { - return composerDist{}, err + return "", err } defer func() { _ = resp.Body.Close() }() h.proxy.Logger.Debug("upstream metadata response", "url", metaURL, "status", resp.StatusCode) - switch { - case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: - return composerDist{}, nil - case resp.StatusCode == http.StatusTooManyRequests && h.proxy.RateLimits != nil: - if err := h.proxy.RateLimits.Held(metaURL); err != nil { - return composerDist{}, err + if resp.StatusCode != http.StatusOK { + return "", nil + } + + var metadata map[string]any + if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { + return "", err + } + + // Expand minified Composer v2 format so that inherited fields (including + // dist) are present on every version entry. Without this, versions that + // inherit dist from a previous entry will appear to have no download URL. + if metadata["minified"] == "composer/2.0" { + h.proxy.Logger.Debug("expanding minified metadata", "url", metaURL) + if packages, ok := metadata["packages"].(map[string]any); ok { + for pkgName, versions := range packages { + versionList, ok := versions.([]any) + if !ok { + continue + } + packages[pkgName] = expandMinifiedVersions(versionList) + } } } - if resp.StatusCode != http.StatusOK { - return composerDist{}, fmt.Errorf("upstream metadata returned %d", resp.StatusCode) - } - body, err := h.proxy.ReadMetadata(resp.Body) - if err != nil { - return composerDist{}, err - } - - dist, err := composerDistURL(body, packageName, version) - if err != nil { - return composerDist{}, err - } + url := h.findDownloadURL(metadata, packageName, version) h.proxy.Logger.Debug("download URL lookup result", "url", metaURL, "package", packageName, "version", version, - "download_url", dist.url) - return dist, nil + "download_url", url) + return url, nil } -// composerDistURL returns the upstream dist of version from Composer -// metadata, with an empty URL when the package has no such version or it has -// no dist URL. It walks the version list in place, expanding only as much of -// the minified format as it needs, since it runs on each archive the proxy -// has not cached yet. -func composerDistURL(body []byte, packageName, version string) (composerDist, error) { - format, _, err := jsonscan.LookupString(body, "minified") - if err != nil { - return composerDist{}, err - } - versions, err := jsonscan.Lookup(body, "packages", packageName) - if err != nil || len(versions) == 0 || versions[0] != '[' { - return composerDist{}, err - } - minified := format == composerMinified - - fields := newComposerFields() - var dist composerDist - err = jsonscan.EachElement(versions, func(entry []byte) error { - if minified && jsonscan.StringIs(entry, composerDevReset) { - fields.reset() - return nil - } - if entry[0] != '{' { - return nil - } - if !minified { - fields.reset() - } - if err := fields.apply(entry, minified); err != nil { - return err - } - if !jsonscan.StringIs(fields.get("version"), version) { - return nil - } - distRaw := fields.get("dist") - dist.url, _, _ = jsonscan.LookupString(distRaw, "url") - if dist.url != "" { - dist.distType, _, _ = jsonscan.LookupString(distRaw, "type") - dist.shasum, _, _ = jsonscan.LookupString(distRaw, "shasum") - dist.published = fields.get("time") - return jsonscan.ErrStop - } - return nil - }) - if errors.Is(err, jsonscan.ErrStop) { - err = nil - } - return dist, err -} - -// composerIntegrity returns the integrity to verify an archive of version -// against: the SHA-1 its dist lists as shasum. A dev branch is left -// unverified: its listed shasum describes only the current head, while every -// commit of the branch is cached under the same version, which holds a single -// recorded integrity, so a cache hit for an older pinned commit would be -// checked against the head's shasum. Composer checks dist.shasum itself -// anyway. A shasum also only describes the listed archive, and Packagist -// lists none for GitHub archives; an unusable one is ignored. Those archives -// are served unverified as before. -func composerIntegrity(shasum, version string, listed bool) string { - if !listed || isDevVersion(version) { +// findDownloadURL finds the dist URL for a specific version in metadata. +func (h *ComposerHandler) findDownloadURL(metadata map[string]any, packageName, version string) string { + packages, ok := metadata["packages"].(map[string]any) + if !ok { return "" } - return sha1Checksum(shasum) -} -// ComposerReleaseArchive returns the file name the Composer handler caches a -// tagged release's archive under, and the integrity it checks it against, -// given the dist the release's metadata lists. It refuses versions the -// handler only serves after reading the metadata again: branches, whose -// archive changes with every push, and archives whose URL has a query, which -// clients ask for with one. -func ComposerReleaseArchive(version, distURL, distType, shasum string) (filename, declared string, err error) { - switch { - case isDevVersion(version): - return "", "", fmt.Errorf("%s is a branch; Composer branches are fetched when requested, not mirrored", version) - case distURL == "": - return "", "", fmt.Errorf("%s has no dist archive", version) - case strings.ContainsAny(distURL, "?#"): - return "", "", fmt.Errorf("the archive URL of %s has a query, which isn't mirrored", version) + versions, ok := packages[packageName].([]any) + if !ok { + return "" } - return distFilename(distURL, distType), composerIntegrity(shasum, version, true), nil -} -// RecordComposerPublishTime stores the publish time of a tagged release from -// the default repository, as the Composer handler does when it resolves the -// release's dist, so the archive is served from the cache without the -// metadata, with or without cooldown. -func (p *Proxy) RecordComposerPublishTime(packageName, version string, publishedAt time.Time) error { - if p.DB == nil || isDevVersion(version) || publishedAt.IsZero() { - return nil + for _, v := range versions { + vmap, ok := v.(map[string]any) + if !ok { + continue + } + + if vmap["version"] == version { + if dist, ok := vmap["dist"].(map[string]any); ok { + if url, ok := dist["url"].(string); ok { + return url + } + } + } } - return p.DB.SetVersionPublishedAt(canonicalVersionPURL("composer", packageName, version), - canonicalPackagePURL("composer", packageName), publishedAt) + + return "" } // proxyUpstream forwards a request to packagist.org without caching. @@ -1080,14 +479,18 @@ func (h *ComposerHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) resp, err := h.proxy.HTTPClient.Do(req) if err != nil { - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return } defer func() { _ = resp.Body.Close() }() - h.proxy.relayResponse(w, r, resp, nil) + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) } diff --git a/internal/handler/composer_advisories.go b/internal/handler/composer_advisories.go deleted file mode 100644 index aa0fa26..0000000 --- a/internal/handler/composer_advisories.go +++ /dev/null @@ -1,351 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "sync" - "time" - - "golang.org/x/sync/singleflight" -) - -const ( - composerAdvisoriesPath = "/api/security-advisories/" - composerFilterSummaryPath = "/lists/summary.json" - composerPackagesParam = "packages" - - // composerAdvisoriesMaxBody caps an advisories query. Composer sends at - // most 500 package names per request. - composerAdvisoriesMaxBody = 1 << 20 -) - -// composerUpstreamIndex holds the parts of the default repository's -// packages.json that the proxy advertises in its own: the security -// advisories API and the filter lists (Packagist's malware list). Composer -// reads advisories and filter entries for each package from its p2 metadata, -// which the proxy passes on unchanged, and asks these URLs for the rest. -type composerUpstreamIndex struct { - advisoriesURL string - advisoriesMetadata bool - filterLists json.RawMessage - filterSummaryURL string -} - -// composerIndexFallbackTTL is how long the upstream index is kept when the -// metadata TTL is 0. -const composerIndexFallbackTTL = time.Hour - -// composerIndexCache keeps the default repository's parsed packages.json in -// memory, whether or not metadata caching is on, so that serving the proxy's -// own packages.json does not wait on the upstream. A failed fetch is -// remembered for the same interval: the last good copy, or none, is used -// until then instead of retrying on every request. The zero value is ready -// to use. -type composerIndexCache struct { - mu sync.Mutex - index composerUpstreamIndex - ok bool - fetchedAt time.Time - group singleflight.Group - now func() time.Time // for tests; nil means time.Now -} - -func (c *composerIndexCache) clock() time.Time { - if c.now != nil { - return c.now() - } - return time.Now() -} - -// upstreamIndex returns the default repository's packages.json fields. ok is -// false when the repository has not been reached since the proxy started; a -// document without the fields yields an empty index. -func (h *ComposerHandler) upstreamIndex(ctx context.Context) (composerUpstreamIndex, bool) { - c := &h.indexCache - ttl := h.proxy.MetadataTTL - if ttl <= 0 { - ttl = composerIndexFallbackTTL - } - c.mu.Lock() - if !c.fetchedAt.IsZero() && c.clock().Sub(c.fetchedAt) < ttl { - index, ok := c.index, c.ok - c.mu.Unlock() - return index, ok - } - c.mu.Unlock() - - // One fetch serves every waiting request. It is detached from the - // request that started it, so a client hanging up neither fails the - // others nor gets remembered as an upstream failure. - _, _, _ = c.group.Do("index", func() (any, error) { - index, err := h.fetchUpstreamIndex(context.WithoutCancel(ctx)) - c.mu.Lock() - defer c.mu.Unlock() - if err == nil { - c.index, c.ok = index, true - } - c.fetchedAt = c.clock() - return nil, nil - }) - c.mu.Lock() - defer c.mu.Unlock() - return c.index, c.ok -} - -// fetchUpstreamIndex fetches and parses the default repository's -// packages.json. A document that doesn't parse is rejected before it is -// cached, so a broken response never replaces a good copy: with metadata -// caching the stored copy is used instead, and without it the error keeps -// the copy in memory. -func (h *ComposerHandler) fetchUpstreamIndex(ctx context.Context) (composerUpstreamIndex, error) { - indexURL := h.repoURL + "/packages.json" - cacheKey := hashedCacheKey(h.repoURL, "/packages.json") - validate := func(body []byte) error { - _, err := parseComposerUpstreamIndex(indexURL, body) - return err - } - res := h.proxy.fetchMetadataDocument(ctx, "composer", cacheKey, indexURL, contentTypeJSON, validate) - if res.err != nil { - h.proxy.Logger.Warn("failed to fetch upstream composer packages.json", "url", indexURL, "error", res.err) - return composerUpstreamIndex{}, res.err - } - return parseComposerUpstreamIndex(indexURL, res.body) -} - -// errComposerIndexNotObject is returned for a packages.json that is valid -// JSON but not an object, such as null. -var errComposerIndexNotObject = errors.New("composer packages.json is not a JSON object") - -// parseComposerUpstreamIndex reads the advisories and filter fields of a -// packages.json. A document without them yields an empty index; one that is -// not a JSON object, or has them in the wrong shape, is an error. -func parseComposerUpstreamIndex(indexURL string, body []byte) (composerUpstreamIndex, error) { - var doc struct { - SecurityAdvisories struct { - Metadata bool `json:"metadata"` - APIURL string `json:"api-url"` - } `json:"security-advisories"` - Filter struct { - Metadata bool `json:"metadata"` - Lists json.RawMessage `json:"lists"` - SummaryURL string `json:"summary-url"` - } `json:"filter"` - } - var index composerUpstreamIndex - if trimmed := bytes.TrimSpace(body); len(trimmed) == 0 || trimmed[0] != '{' { - return index, errComposerIndexNotObject - } - if err := json.Unmarshal(body, &doc); err != nil { - return index, fmt.Errorf("parsing composer packages.json: %w", err) - } - // Composer refuses advisories configured without an api-url unless the - // repository lists its packages, which the proxy does not. - if apiURL, ok := resolveComposerURL(indexURL, doc.SecurityAdvisories.APIURL); ok { - index.advisoriesURL = apiURL - index.advisoriesMetadata = doc.SecurityAdvisories.Metadata - } - // Composer only applies filter lists a repository marks with metadata. - // A filter api-url is not passed on: without it Composer reads the - // entries from package metadata, narrowed by the summary. - if doc.Filter.Metadata && len(doc.Filter.Lists) > 0 && doc.Filter.Lists[0] == '{' { - index.filterLists = doc.Filter.Lists - index.filterSummaryURL, _ = resolveComposerURL(indexURL, doc.Filter.SummaryURL) - } - return index, nil -} - -// resolveComposerURL resolves a URL from packages.json against the document's -// own, as Composer does, and accepts only http and https. -func resolveComposerURL(base, ref string) (string, bool) { - if ref == "" { - return "", false - } - b, err := url.Parse(base) - if err != nil { - return "", false - } - u, err := b.Parse(ref) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { - return "", false - } - return u.String(), true -} - -// advertise adds the advisories and filter entries to the proxy's -// packages.json, pointing at the proxy. -func (index composerUpstreamIndex) advertise(packages map[string]any, proxyURL string) { - if index.advisoriesURL != "" { - packages["security-advisories"] = map[string]any{ - "metadata": index.advisoriesMetadata, - "api-url": proxyURL + "/composer" + composerAdvisoriesPath, - } - } - if index.filterLists != nil { - filter := map[string]any{"metadata": true, "lists": index.filterLists} - if index.filterSummaryURL != "" { - filter["summary-url"] = proxyURL + "/composer" + composerFilterSummaryPath - } - packages["filter"] = filter - } -} - -// handleSecurityAdvisories relays a security advisories query to the URL the -// default repository advertises. Composer sends the package names as -// packages[] in a POST form; a GET query is accepted too. Names served by a -// package route or a hosted-only vendor are removed first, so private package -// names are never sent to the public API. -// -// Responses are not cached: each query names a different set of packages. -func (h *ComposerHandler) handleSecurityAdvisories(w http.ResponseWriter, r *http.Request) { - query, err := h.advisoriesQuery(w, r) - if err != nil { - var maxBytesErr *http.MaxBytesError - if errors.As(err, &maxBytesErr) { - JSONError(w, http.StatusRequestEntityTooLarge, "advisories request too large") - return - } - JSONError(w, http.StatusBadRequest, "could not read advisories request") - return - } - - asked := len(query[composerPackagesParam+"[]"]) - public := h.publicPackageNames(query[composerPackagesParam+"[]"]) - if asked > 0 && len(public) == 0 { - w.Header().Set(headerContentType, contentTypeJSON) - _, _ = io.WriteString(w, `{"advisories":{}}`) - return - } - if asked > 0 { - query[composerPackagesParam+"[]"] = public - } - - index, ok := h.upstreamIndex(r.Context()) - if !ok { - JSONError(w, http.StatusBadGateway, "upstream repository unavailable") - return - } - if index.advisoriesURL == "" { - JSONError(w, http.StatusNotFound, "upstream repository has no security advisories API") - return - } - - req, err := h.advisoriesRequest(r, index.advisoriesURL, query) - if err != nil { - JSONError(w, http.StatusInternalServerError, "failed to create request") - return - } - h.proxy.Logger.Debug("composer advisories request", "packages", len(public), "withheld", asked-len(public)) - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("composer advisories request failed", "error", err) - JSONError(w, http.StatusBadGateway, "failed to reach upstream repository") - return - } - defer func() { _ = resp.Body.Close() }() - - h.proxy.relayResponse(w, r, resp, nil) -} - -// advisoriesQuery reads the query of an advisories request, from the form -// body of a POST or the URL of a GET. Package names come back under -// "packages[]" whichever spelling the client used: PHP's http_build_query, -// which Composer uses, numbers them as packages[0], packages[1], ... -func (h *ComposerHandler) advisoriesQuery(w http.ResponseWriter, r *http.Request) (url.Values, error) { - raw := r.URL.RawQuery - if r.Method == http.MethodPost { - body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, composerAdvisoriesMaxBody)) - if err != nil { - return nil, err - } - raw = string(body) - } - values, err := url.ParseQuery(raw) - if err != nil { - return nil, err - } - query := url.Values{} - for key, vals := range values { - if key == composerPackagesParam || strings.HasPrefix(key, composerPackagesParam+"[") { - key = composerPackagesParam + "[]" - } - query[key] = append(query[key], vals...) - } - return query, nil -} - -// publicPackageNames drops the names the default repository does not serve. -func (h *ComposerHandler) publicPackageNames(names []string) []string { - public := make([]string, 0, len(names)) - for _, name := range names { - if _, routed := h.routes.match(name); routed || h.proxy.hostedOnly("composer", name) { - continue - } - public = append(public, name) - } - return public -} - -// advisoriesRequest builds the upstream request, sending the query the way -// the client did. -func (h *ComposerHandler) advisoriesRequest(r *http.Request, apiURL string, query url.Values) (*http.Request, error) { - u, err := url.Parse(apiURL) - if err != nil { - return nil, err - } - var body io.Reader - if r.Method == http.MethodPost { - body = strings.NewReader(query.Encode()) - } else { - u.RawQuery = joinQuery(u.RawQuery, query.Encode()) - } - req, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), body) - if err != nil { - return nil, err - } - if r.Method == http.MethodPost { - req.Header.Set(headerContentType, "application/x-www-form-urlencoded") - } - for _, header := range []string{headerAccept, headerAcceptEncoding} { - if v := r.Header.Get(header); v != "" { - req.Header.Set(header, v) - } - } - return req, nil -} - -func joinQuery(a, b string) string { - if a == "" || b == "" { - return a + b - } - return a + "&" + b -} - -// handleFilterSummary relays the filter list summary the default repository -// advertises. It lists every package on a filter list, so the request names -// no packages. It is not cached, so a newly listed package is blocked as soon -// as the upstream lists it; Composer revalidates its own copy with -// If-Modified-Since, which is passed on. -func (h *ComposerHandler) handleFilterSummary(w http.ResponseWriter, r *http.Request) { - index, ok := h.upstreamIndex(r.Context()) - if !ok { - JSONError(w, http.StatusBadGateway, "upstream repository unavailable") - return - } - if index.filterSummaryURL == "" { - JSONError(w, http.StatusNotFound, "upstream repository has no filter list summary") - return - } - h.proxy.ProxyUpstream(w, r, index.filterSummaryURL, - []string{headerAccept, headerAcceptEncoding, "If-Modified-Since", "If-None-Match"}) -} diff --git a/internal/handler/composer_advisories_test.go b/internal/handler/composer_advisories_test.go deleted file mode 100644 index 097d8cc..0000000 --- a/internal/handler/composer_advisories_test.go +++ /dev/null @@ -1,470 +0,0 @@ -package handler - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "net/url" - "slices" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/database" -) - -// composerAdvisoryUpstream serves a packages.json advertising security -// advisories and a malware filter list, and records the advisories and -// summary requests it gets. -type composerAdvisoryUpstream struct { - *httptest.Server - index string - down atomic.Bool // answer every request with 503 - // malformed answers packages.json with an HTML page and status 200. - malformed atomic.Bool - - mu sync.Mutex - requests []*recordedRequest -} - -type recordedRequest struct { - method string - path string - query url.Values - body string - header http.Header -} - -const composerAdvisoriesResponse = `{"advisories":{"public/library":[{"advisoryId":"PKSA-1","affectedVersions":"<1.0.1"}]}}` - -const composerFilterSummary = `{"filter":{"malware":{"evil/package":"1.0.0"}}}` - -func newComposerAdvisoryUpstream(t *testing.T) *composerAdvisoryUpstream { - t.Helper() - u := &composerAdvisoryUpstream{} - u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - u.mu.Lock() - u.requests = append(u.requests, &recordedRequest{r.Method, r.URL.Path, r.URL.Query(), string(body), r.Header.Clone()}) - u.mu.Unlock() - if u.down.Load() { - w.WriteHeader(http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", "application/json") - switch r.URL.Path { - case "/packages.json": - if u.malformed.Load() { - w.Header().Set("Content-Type", "text/html") - _, _ = io.WriteString(w, "Service temporarily unavailable") - return - } - _, _ = io.WriteString(w, u.index) - case "/api/security-advisories/": - _, _ = io.WriteString(w, composerAdvisoriesResponse) - case "/lists/all/summary.json": - if r.Header.Get("If-Modified-Since") != "" { - w.WriteHeader(http.StatusNotModified) - return - } - _, _ = io.WriteString(w, composerFilterSummary) - default: - http.NotFound(w, r) - } - })) - // Like Packagist's: the API on one host, the summary as a relative URL. - u.index = `{ - "packages": [], - "metadata-url": "/p2/%package%.json", - "security-advisories": {"metadata": true, "api-url": "` + u.URL + `/api/security-advisories/"}, - "filter": {"metadata": true, "lists": {"malware": {"enabled": true}}, "summary-url": "/lists/all/summary.json"} - }` - t.Cleanup(u.Close) - return u -} - -// requestsTo returns the recorded requests for path. -func (u *composerAdvisoryUpstream) requestsTo(path string) []*recordedRequest { - u.mu.Lock() - defer u.mu.Unlock() - var out []*recordedRequest - for _, r := range u.requests { - if r.path == path { - out = append(out, r) - } - } - return out -} - -func newAdvisoryComposerHandler(t *testing.T, upstream *composerAdvisoryUpstream) *ComposerHandler { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.HostedEnabled = true - proxy.HostedComposerVendors = []string{"acme"} - return NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL). - WithPackageRoutes(map[string]string{"example/*": "https://packages.example.com/composer"}) -} - -func serveComposerIndex(t *testing.T, h *ComposerHandler) map[string]any { - t.Helper() - w := serveComposer(h, "/packages.json") - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body) - } - var index map[string]any - if err := json.Unmarshal(w.Body.Bytes(), &index); err != nil { - t.Fatalf("parsing packages.json: %v", err) - } - return index -} - -func TestComposerServiceIndexAdvertisesAdvisoriesAndFilter(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - index := serveComposerIndex(t, newAdvisoryComposerHandler(t, upstream)) - - advisories, _ := index["security-advisories"].(map[string]any) - if advisories["api-url"] != "http://proxy.test/composer/api/security-advisories/" || advisories["metadata"] != true { - t.Errorf("security-advisories = %v, want the proxy's api-url with metadata", index["security-advisories"]) - } - filter, _ := index["filter"].(map[string]any) - if filter["summary-url"] != "http://proxy.test/composer/lists/summary.json" || filter["metadata"] != true { - t.Errorf("filter = %v, want the proxy's summary-url with metadata", index["filter"]) - } - lists, _ := filter["lists"].(map[string]any) - if _, ok := lists["malware"]; !ok { - t.Errorf("filter lists = %v, want upstream's malware list", filter["lists"]) - } - if index["metadata-url"] != "http://proxy.test/composer/p2/%package%.json" { - t.Errorf("metadata-url = %v, want the proxy's", index["metadata-url"]) - } -} - -func TestComposerServiceIndexWithoutUpstreamSupport(t *testing.T) { - tests := []struct { - name string - index string - }{ - {"not advertised", `{"packages": [], "metadata-url": "/p2/%package%.json"}`}, - {"advisories without api-url", `{"security-advisories": {"metadata": true}}`}, - {"filter without metadata", `{"filter": {"lists": {"malware": {"enabled": true}}, "api-url": "/filter"}}`}, - {"not json", ``}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - upstream.index = tt.index - index := serveComposerIndex(t, newAdvisoryComposerHandler(t, upstream)) - for _, key := range []string{"security-advisories", "filter"} { - if _, ok := index[key]; ok { - t.Errorf("%s advertised although upstream does not offer it: %v", key, index[key]) - } - } - }) - } -} - -func TestComposerServiceIndexUpstreamDown(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - h := newAdvisoryComposerHandler(t, upstream) - upstream.Close() - - index := serveComposerIndex(t, h) - if _, ok := index["security-advisories"]; ok { - t.Errorf("security-advisories advertised without an upstream index: %v", index["security-advisories"]) - } - if index["metadata-url"] == nil { - t.Error("packages.json lacks metadata-url while the upstream is down") - } -} - -func advertisesAdvisories(index map[string]any) bool { - _, ok := index["security-advisories"] - return ok -} - -func TestComposerServiceIndexUpstreamDownNotRetried(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - upstream.down.Store(true) - h := newAdvisoryComposerHandler(t, upstream) - - serveComposerIndex(t, h) - fetched := len(upstream.requestsTo("/packages.json")) - if fetched == 0 { - t.Fatal("first packages.json made no upstream request") - } - index := serveComposerIndex(t, h) - if got := len(upstream.requestsTo("/packages.json")); got != fetched { - t.Errorf("second packages.json made %d upstream requests, want none", got-fetched) - } - if advertisesAdvisories(index) { - t.Errorf("security-advisories advertised without an upstream index: %v", index["security-advisories"]) - } -} - -func TestComposerServiceIndexKeptInMemory(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - h := newAdvisoryComposerHandler(t, upstream) - h.proxy.MetadataTTL = 0 - now := time.Now() - h.indexCache.now = func() time.Time { return now } - - if !advertisesAdvisories(serveComposerIndex(t, h)) { - t.Fatal("security-advisories not advertised") - } - if got := len(upstream.requestsTo("/packages.json")); got != 1 { - t.Fatalf("upstream packages.json requests = %d, want 1", got) - } - - // Within the interval the copy in memory is served without asking. - upstream.down.Store(true) - now = now.Add(composerIndexFallbackTTL - time.Minute) - if !advertisesAdvisories(serveComposerIndex(t, h)) { - t.Error("security-advisories dropped within the interval") - } - if got := len(upstream.requestsTo("/packages.json")); got != 1 { - t.Errorf("upstream packages.json requests = %d within the interval, want 1", got) - } - - // After it the index is fetched again; the failure keeps the last good - // copy and is not retried until the next interval. - now = now.Add(2 * time.Minute) - if !advertisesAdvisories(serveComposerIndex(t, h)) { - t.Error("last good copy dropped after a failed fetch") - } - fetched := len(upstream.requestsTo("/packages.json")) - if fetched < 2 { - t.Fatalf("upstream packages.json requests = %d after the interval, want a new fetch", fetched) - } - serveComposerIndex(t, h) - if got := len(upstream.requestsTo("/packages.json")); got != fetched { - t.Errorf("failed fetch retried %d times within the interval", got-fetched) - } -} - -// assertComposerSecurityAdvertised checks that h advertises the upstream's -// advisories API and malware filter, and relays both. -func assertComposerSecurityAdvertised(t *testing.T, h *ComposerHandler, when string) { - t.Helper() - index := serveComposerIndex(t, h) - if !advertisesAdvisories(index) { - t.Errorf("%s: security-advisories not advertised: %v", when, index) - } - if _, ok := index["filter"]; !ok { - t.Errorf("%s: filter not advertised: %v", when, index) - } - w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B%5D=public%2Flibrary") - if w.Code != http.StatusOK || w.Body.String() != composerAdvisoriesResponse { - t.Errorf("%s: advisories = %d %q, want upstream's advisories", when, w.Code, w.Body) - } - if w := serveComposer(h, "/lists/summary.json"); w.Code != http.StatusOK || w.Body.String() != composerFilterSummary { - t.Errorf("%s: filter summary = %d %q, want upstream's summary", when, w.Code, w.Body) - } -} - -// A packages.json that doesn't parse, sent with status 200, must not replace -// the last good index, with metadata caching on or off: the advisories API -// and the malware filter stay advertised and relayed. -func TestComposerServiceIndexMalformedRefreshKeepsLastGood(t *testing.T) { - for _, cacheMetadata := range []bool{false, true} { - t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - h := newAdvisoryComposerHandler(t, upstream) - h.proxy.CacheMetadata = cacheMetadata - h.proxy.MetadataTTL = time.Hour - now := time.Now() - h.indexCache.now = func() time.Time { return now } - cacheKey := hashedCacheKey(upstream.URL, "/packages.json") - - assertComposerSecurityAdvertised(t, h, "before the refresh") - var stored *database.MetadataCacheEntry - if cacheMetadata { - var err error - if stored, err = h.proxy.DB.GetMetadataCache("composer", cacheKey); err != nil || stored == nil { - t.Fatalf("packages.json not cached: %v, %v", stored, err) - } - expireMetadata(t, h.proxy.DB, "composer", cacheKey) - } - - upstream.malformed.Store(true) - now = now.Add(2 * time.Hour) - fetched := len(upstream.requestsTo("/packages.json")) - assertComposerSecurityAdvertised(t, h, "after a malformed refresh") - if got := len(upstream.requestsTo("/packages.json")); got == fetched { - t.Fatal("packages.json was not fetched again after the TTL") - } - - if !cacheMetadata { - return - } - after, err := h.proxy.DB.GetMetadataCache("composer", cacheKey) - if err != nil || after == nil || after.ContentDigest != stored.ContentDigest { - t.Errorf("cached packages.json replaced by the malformed refresh: %v, %v", after, err) - } - // A fresh handler, as after a restart, has nothing in memory and - // reads the stored copy. - restarted := NewComposerHandlerWithUpstreams(h.proxy, "http://proxy.test", upstream.URL, upstream.URL) - assertComposerSecurityAdvertised(t, restarted, "after a restart") - }) - } -} - -func TestComposerServiceIndexFetchedOnce(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - h := newAdvisoryComposerHandler(t, upstream) - - var wg sync.WaitGroup - for range 10 { - wg.Go(func() { serveComposer(h, "/packages.json") }) - } - wg.Wait() - if got := len(upstream.requestsTo("/packages.json")); got != 1 { - t.Errorf("upstream packages.json requests = %d for parallel requests, want 1", got) - } -} - -func serveAdvisories(h *ComposerHandler, method, target, body string) *httptest.ResponseRecorder { - var reader io.Reader - if body != "" { - reader = strings.NewReader(body) - } - req := httptest.NewRequest(method, target, reader) - if method == http.MethodPost { - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - } - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - return w -} - -func TestComposerAdvisoriesWithholdPrivateNames(t *testing.T) { - // The form Composer sends: PHP's http_build_query numbers the entries. - form := url.Values{ - "packages[0]": {"public/library"}, - "packages[1]": {"example/private"}, - "packages[2]": {"Acme/hosted"}, - "packages[3]": {"other/library"}, - } - tests := []struct { - name, method, target, body string - }{ - {"POST form", http.MethodPost, "/api/security-advisories/", form.Encode()}, - {"GET query", http.MethodGet, "/api/security-advisories/?" + form.Encode(), ""}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - h := newAdvisoryComposerHandler(t, upstream) - - w := serveAdvisories(h, tt.method, tt.target, tt.body) - - if w.Code != http.StatusOK || w.Body.String() != composerAdvisoriesResponse { - t.Fatalf("got %d %q, want upstream's advisories", w.Code, w.Body) - } - sent := upstream.requestsTo("/api/security-advisories/") - if len(sent) != 1 { - t.Fatalf("upstream got %d advisories requests, want 1", len(sent)) - } - if sent[0].method != tt.method { - t.Errorf("upstream method = %s, want %s", sent[0].method, tt.method) - } - query := sent[0].query - if tt.method == http.MethodPost { - query, _ = url.ParseQuery(sent[0].body) - } - got := query["packages[]"] - slices.Sort(got) - if want := []string{"other/library", "public/library"}; !slices.Equal(got, want) { - t.Errorf("upstream was asked about %v, want only the public packages %v", got, want) - } - }) - } -} - -func TestComposerAdvisoriesOnlyPrivateNames(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - h := newAdvisoryComposerHandler(t, upstream) - - w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B0%5D=example%2Fprivate") - - if w.Code != http.StatusOK || w.Body.String() != `{"advisories":{}}` { - t.Errorf("got %d %q, want an empty advisories answer", w.Code, w.Body) - } - if sent := upstream.requestsTo("/api/security-advisories/"); len(sent) != 0 { - t.Errorf("upstream was asked although every package is private: %+v", sent[0]) - } -} - -func TestComposerAdvisoriesUpdatedSince(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - h := newAdvisoryComposerHandler(t, upstream) - - w := serveAdvisories(h, http.MethodGet, "/api/security-advisories/?updatedSince=1700000000", "") - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200", w.Code) - } - sent := upstream.requestsTo("/api/security-advisories/") - if len(sent) != 1 || sent[0].query.Get("updatedSince") != "1700000000" { - t.Errorf("updatedSince was not passed on: %+v", sent) - } -} - -func TestComposerAdvisoriesNotOffered(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - upstream.index = `{"packages": []}` - h := newAdvisoryComposerHandler(t, upstream) - - if w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B%5D=public%2Flibrary"); w.Code != http.StatusNotFound { - t.Errorf("advisories status = %d, want 404 when upstream has no API", w.Code) - } - if w := serveComposer(h, "/lists/summary.json"); w.Code != http.StatusNotFound { - t.Errorf("summary status = %d, want 404 when upstream has no filter", w.Code) - } -} - -func TestComposerFilterSummaryRelayed(t *testing.T) { - upstream := newComposerAdvisoryUpstream(t) - h := newAdvisoryComposerHandler(t, upstream) - - w := serveComposer(h, "/lists/summary.json") - if w.Code != http.StatusOK || w.Body.String() != composerFilterSummary { - t.Fatalf("got %d %q, want upstream's summary", w.Code, w.Body) - } - - req := httptest.NewRequest(http.MethodGet, "/lists/summary.json", nil) - req.Header.Set("If-Modified-Since", "Wed, 21 Oct 2026 15:45:01 GMT") - w = httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusNotModified { - t.Errorf("revalidation status = %d, want 304 passed on from upstream", w.Code) - } -} - -func TestComposerMetadataKeepsAdvisoriesAndFilter(t *testing.T) { - h := NewComposerHandler(testProxy(), "http://proxy.test") - body := []byte(`{ - "minified": "composer/2.0", - "packages": {"evil/package": [{"name": "evil/package", "version": "1.0.0", "dist": {"url": "https://example.com/evil.zip", "type": "zip"}}]}, - "security-advisories": [{"advisoryId": "PKSA-1", "affectedVersions": "<2.0.0"}], - "filter": {"malware": [{"constraint": "1.0.0", "reason": "malware", "id": "PKFE-1"}]} - }`) - - out, _, err := h.rewriteMetadataKeeping(body) - if err != nil { - t.Fatalf("rewrite: %v", err) - } - var doc map[string]json.RawMessage - if err := json.Unmarshal(out, &doc); err != nil { - t.Fatalf("parsing rewritten metadata: %v", err) - } - if !strings.Contains(string(doc["security-advisories"]), "PKSA-1") { - t.Errorf("security-advisories lost in rewrite: %s", out) - } - if !strings.Contains(string(doc["filter"]), "PKFE-1") { - t.Errorf("filter lost in rewrite: %s", out) - } -} diff --git a/internal/handler/composer_download_lookup_test.go b/internal/handler/composer_download_lookup_test.go deleted file mode 100644 index 53a0cfe..0000000 --- a/internal/handler/composer_download_lookup_test.go +++ /dev/null @@ -1,356 +0,0 @@ -package handler - -import ( - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - registryclient "github.com/git-pkgs/registries/client" - "github.com/git-pkgs/registries/fetch" -) - -// composerLookupUpstream serves v/p's metadata listing versions and the -// archives they point at, counts metadata and archive requests, and fails -// every request while down is set, with downStatus or 503. A version listed in -// published carries that publish time. -type composerLookupUpstream struct { - h http.Handler - handler *ComposerHandler - proxy *Proxy - versions atomic.Value // []string - published map[string]time.Time - down atomic.Bool - downStatus atomic.Int32 - metadata atomic.Int32 - archives atomic.Int32 - routeURL string -} - -func setupComposerLookup(t *testing.T, cacheMetadata bool, versions ...string) *composerLookupUpstream { - t.Helper() - u := &composerLookupUpstream{published: map[string]time.Time{}} - u.versions.Store(versions) - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if u.down.Load() { - status := int(u.downStatus.Load()) - if status == 0 { - status = http.StatusServiceUnavailable - } - w.Header().Set("Retry-After", "30") - http.Error(w, "down", status) - return - } - switch { - case r.URL.Path == "/p2/v/p.json": - u.metadata.Add(1) - var entries []string - for _, v := range u.versions.Load().([]string) { - var published string - if at, ok := u.published[v]; ok { - published = fmt.Sprintf(`"time":%q,`, at.Format(time.RFC3339)) - } - entries = append(entries, fmt.Sprintf(`{"version":%q,%s"dist":{"url":%q,"type":"zip"}}`, v, published, srv.URL+"/archives/"+v+".zip")) - } - _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[%s]}}`, strings.Join(entries, ",")) - case r.URL.Path == "/p2/v/p~dev.json": - u.metadata.Add(1) - http.NotFound(w, r) - case strings.HasPrefix(r.URL.Path, "/archives/"): - u.archives.Add(1) - _, _ = io.WriteString(w, "archive "+r.URL.Path) - default: - http.NotFound(w, r) - } - })) - t.Cleanup(srv.Close) - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = cacheMetadata - proxy.MetadataTTL = time.Hour - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - u.proxy = proxy - u.handler = NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL) - u.h = u.handler.Routes() - u.routeURL = srv.URL - return u -} - -// route sends v/p to a custom repository, which is the same server under -// another name. -func (u *composerLookupUpstream) route() { - u.h = u.handler.WithPackageRoutes(map[string]string{"v/*": u.routeURL}).Routes() -} - -func (u *composerLookupUpstream) get(t *testing.T, target string) *httptest.ResponseRecorder { - t.Helper() - w := httptest.NewRecorder() - u.h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - return w -} - -func (u *composerLookupUpstream) download(t *testing.T, version string) { - t.Helper() - w := u.get(t, "/files/v/p/"+version+"/"+version+".zip") - if want := "archive /archives/" + version + ".zip"; w.Code != http.StatusOK || w.Body.String() != want { - t.Fatalf("download %s: status %d, body %q, want 200 %q", version, w.Code, w.Body.String(), want) - } -} - -// TestComposerDownloadResolvesFromCachedMetadata checks that downloads find -// their archive in the metadata Composer was just served, instead of fetching -// it from upstream again for every archive. -func TestComposerDownloadResolvesFromCachedMetadata(t *testing.T) { - u := setupComposerLookup(t, true, "1.0.0") - if w := u.get(t, "/p2/v/p.json"); w.Code != http.StatusOK { - t.Fatalf("metadata: status %d", w.Code) - } - u.download(t, "1.0.0") - u.download(t, "1.0.0") - if n := u.metadata.Load(); n != 1 { - t.Errorf("upstream metadata requests = %d, want 1 (the metadata request itself)", n) - } - if n := u.archives.Load(); n != 1 { - t.Errorf("upstream archive requests = %d, want 1", n) - } -} - -// TestComposerCachedArchiveServedWhileUpstreamDown checks that a cached -// archive stays installable when upstream fails, rather than failing on the -// metadata lookup that only names where to fetch it from. -func TestComposerCachedArchiveServedWhileUpstreamDown(t *testing.T) { - u := setupComposerLookup(t, true, "1.0.0") - u.download(t, "1.0.0") - u.down.Store(true) - u.download(t, "1.0.0") -} - -// TestComposerDownloadOfVersionNewerThanCachedMetadata checks that a version -// published after the metadata was cached, which a lock file can name before -// the cached copy expires, is still looked up upstream. -func TestComposerDownloadOfVersionNewerThanCachedMetadata(t *testing.T) { - u := setupComposerLookup(t, true, "1.0.0") - u.download(t, "1.0.0") - u.versions.Store([]string{"1.0.0", "1.1.0"}) - u.download(t, "1.1.0") -} - -// TestComposerDownloadWithoutMetadataCache checks that without cache_metadata -// a download asks upstream for the metadata once, as it always has. -func TestComposerDownloadWithoutMetadataCache(t *testing.T) { - u := setupComposerLookup(t, false, "1.0.0") - u.download(t, "1.0.0") - if n := u.metadata.Load(); n != 1 { - t.Errorf("upstream metadata requests = %d, want 1", n) - } -} - -// TestComposerDownloadLookupUpstreamError checks that a download whose -// metadata lookup fails upstream answers 502, or 503 while upstream holds -// requests, and not 404: Composer treats 404 as the version being gone. -func TestComposerDownloadLookupUpstreamError(t *testing.T) { - for _, status := range []int{http.StatusInternalServerError, http.StatusServiceUnavailable} { - t.Run(http.StatusText(status), func(t *testing.T) { - u := setupComposerLookup(t, false, "1.0.0") - u.downStatus.Store(int32(status)) - u.down.Store(true) - if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusBadGateway { - t.Errorf("status = %d, want 502", w.Code) - } - }) - } - - t.Run("rate limited", func(t *testing.T) { - u := setupComposerLookup(t, false, "1.0.0") - gate := registryclient.NewRateLimitGate(http.DefaultTransport) - u.proxy.HTTPClient = &http.Client{Transport: gate} - u.proxy.RateLimits = gate - u.downStatus.Store(http.StatusTooManyRequests) - u.down.Store(true) - w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip") - if w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" { - t.Errorf("status = %d, Retry-After %q; want 503 with Retry-After", w.Code, w.Header().Get("Retry-After")) - } - }) - - t.Run("not listed", func(t *testing.T) { - u := setupComposerLookup(t, false, "1.0.0") - if w := u.get(t, "/files/v/p/2.0.0/2.0.0.zip"); w.Code != http.StatusNotFound { - t.Errorf("status = %d, want 404", w.Code) - } - }) -} - -// storedPublishTime returns the publish time recorded for v/p at version. -func (u *composerLookupUpstream) storedPublishTime(t *testing.T, version string) (time.Time, bool) { - t.Helper() - ver, err := u.proxy.DB.GetVersionByPURL(canonicalVersionPURL("composer", "v/p", version)) - if err != nil { - t.Fatalf("reading versions row: %v", err) - } - if ver == nil || !ver.PublishedAt.Valid { - return time.Time{}, false - } - return ver.PublishedAt.Time, true -} - -func (u *composerLookupUpstream) wantMetadataRequests(t *testing.T, want int32) { - t.Helper() - if n := u.metadata.Load(); n != want { - t.Errorf("upstream metadata requests = %d, want %d", n, want) - } -} - -// TestComposerCachedReleaseSkipsMetadata checks that a tagged release whose -// archive is cached is served without asking upstream for the metadata -// again, with and without cache_metadata, and with cooldown, which takes the -// publish time recorded when the archive was first resolved. -func TestComposerCachedReleaseSkipsMetadata(t *testing.T) { - for _, tt := range []struct { - name string - cacheMetadata bool - cooldown bool - }{ - {"metadata not cached", false, false}, - {"metadata cached", true, false}, - {"cooldown", false, true}, - } { - t.Run(tt.name, func(t *testing.T) { - u := setupComposerLookup(t, tt.cacheMetadata, "1.0.0") - u.published["1.0.0"] = time.Now().Add(-30 * 24 * time.Hour) - if tt.cooldown { - u.proxy.Cooldown = &cooldown.Config{Default: "3d"} - } - u.download(t, "1.0.0") - // With cache_metadata the cached copy expires; the archive must - // not need it. - u.proxy.MetadataTTL = 0 - u.download(t, "1.0.0") - u.download(t, "1.0.0") - u.wantMetadataRequests(t, 1) - if n := u.archives.Load(); n != 1 { - t.Errorf("upstream archive requests = %d, want 1", n) - } - }) - } -} - -// TestComposerDownloadStoresPublishTime checks that resolving a tagged -// release records its publish time, so later downloads can check cooldown -// without the metadata. -func TestComposerDownloadStoresPublishTime(t *testing.T) { - u := setupComposerLookup(t, false, "1.0.0") - published := time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC) - u.published["1.0.0"] = published - u.download(t, "1.0.0") - got, ok := u.storedPublishTime(t, "1.0.0") - if !ok || !got.Equal(published) { - t.Errorf("stored publish time = %v, %v, want %v", got, ok, published) - } -} - -// TestComposerCachedReleaseCooldownFromStoredTime checks that a cached -// archive whose stored publish time is still inside the cooldown window is -// withheld without the metadata, including when cooldown was enabled or -// raised after the archive was cached. -func TestComposerCachedReleaseCooldownFromStoredTime(t *testing.T) { - u := setupComposerLookup(t, false, "1.0.0", "2.0.0") - u.published["1.0.0"] = time.Now().Add(-5 * 24 * time.Hour) - u.published["2.0.0"] = time.Now().Add(-1 * time.Hour) - u.download(t, "2.0.0") - u.proxy.Cooldown = &cooldown.Config{Default: "3d"} - u.download(t, "1.0.0") - - if w := u.get(t, "/files/v/p/2.0.0/2.0.0.zip"); w.Code != http.StatusNotFound { - t.Errorf("2.0.0 after enabling cooldown: status %d, want 404", w.Code) - } - u.proxy.Cooldown = &cooldown.Config{Default: "7d"} - if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusNotFound { - t.Errorf("1.0.0 after raising cooldown: status %d, want 404", w.Code) - } - u.wantMetadataRequests(t, 2) -} - -// TestComposerCachedReleaseWithoutStoredTime checks that with cooldown a -// cached archive the proxy recorded no publish time for, such as one cached -// by an older version, is checked against the metadata, which then records -// the time. -func TestComposerCachedReleaseWithoutStoredTime(t *testing.T) { - u := setupComposerLookup(t, false, "1.0.0") - u.published["1.0.0"] = time.Now().Add(-1 * time.Hour) - u.download(t, "1.0.0") - if _, err := u.proxy.DB.Exec(`UPDATE versions SET published_at = NULL`); err != nil { - t.Fatal(err) - } - - u.proxy.Cooldown = &cooldown.Config{Default: "3d"} - if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusNotFound { - t.Errorf("status %d, want 404", w.Code) - } - u.wantMetadataRequests(t, 2) - if _, ok := u.storedPublishTime(t, "1.0.0"); !ok { - t.Error("publish time not recorded from the metadata") - } -} - -// TestComposerCachedDevVersionUsesMetadata checks that a branch, whose -// archive and publish time change with every push, is still resolved from -// the metadata on each download and records no publish time. -func TestComposerCachedDevVersionUsesMetadata(t *testing.T) { - u := setupComposerLookup(t, false, "dev-main") - u.published["dev-main"] = time.Now().Add(-30 * 24 * time.Hour) - u.download(t, "dev-main") - before := u.metadata.Load() - u.download(t, "dev-main") - if u.metadata.Load() == before { - t.Error("cached branch archive served without consulting the metadata") - } - if _, ok := u.storedPublishTime(t, "dev-main"); ok { - t.Error("publish time recorded for a branch") - } -} - -// TestComposerCachedRoutedReleaseUsesMetadata checks that a routed package -// keeps resolving every download from its route's metadata: the versions -// table is keyed by a PURL that does not say which repository the version -// came from, so it neither stores nor trusts a publish time for it. -func TestComposerCachedRoutedReleaseUsesMetadata(t *testing.T) { - u := setupComposerLookup(t, false, "1.0.0") - u.published["1.0.0"] = time.Now().Add(-1 * time.Hour) - u.route() - u.download(t, "1.0.0") - u.download(t, "1.0.0") - u.wantMetadataRequests(t, 2) - if _, ok := u.storedPublishTime(t, "1.0.0"); ok { - t.Error("publish time recorded for a routed package") - } - - // A time stored for the public package of the same name is not used. - if err := u.proxy.DB.SetVersionPublishedAt(canonicalVersionPURL("composer", "v/p", "1.0.0"), - canonicalPackagePURL("composer", "v/p"), time.Now().Add(-30*24*time.Hour)); err != nil { - t.Fatal(err) - } - u.proxy.Cooldown = &cooldown.Config{Default: "3d"} - if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusNotFound { - t.Errorf("routed release in cooldown: status %d, want 404", w.Code) - } -} - -// TestComposerCachedReleaseDenied checks that the denylist still blocks a -// cached release served without the metadata. -func TestComposerCachedReleaseDenied(t *testing.T) { - u := setupComposerLookup(t, false, "1.0.0") - u.download(t, "1.0.0") - setTestDenylist(t, u.proxy, "pkg:composer/v/p@1.0.0") - if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusForbidden { - t.Errorf("status %d, want 403", w.Code) - } -} diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go deleted file mode 100644 index be8a1d6..0000000 --- a/internal/handler/composer_hosted.go +++ /dev/null @@ -1,134 +0,0 @@ -package handler - -import ( - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "strings" - "time" -) - -// composerRootOnlyKeys are composer.json keys that only apply to the root -// project. Repositories leave them out of the version entries they serve. -var composerRootOnlyKeys = []string{"repositories", "config", "minimum-stability", "prefer-stable"} - -// composerLocalEntry is a pushed version as Composer metadata lists it. -type composerLocalEntry struct { - version string - entry json.RawMessage -} - -// composerLocalEntries returns the pushed versions of a package for one of -// its metadata files: the dev versions for the ~dev file, the tagged ones for -// the other. -func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([]composerLocalEntry, error) { - local, err := h.proxy.localVersions("composer", packageName) - if err != nil || len(local) == 0 { - return nil, err - } - var entries []composerLocalEntry - for _, lv := range local { - if isDevVersion(lv.Version) != dev { - continue - } - var manifest map[string]any - if err := json.Unmarshal(lv.Manifest, &manifest); err != nil { - return nil, fmt.Errorf("decoding manifest of %s %s: %w", packageName, lv.Version, err) - } - for _, key := range composerRootOnlyKeys { - delete(manifest, key) - } - manifest["name"] = packageName - manifest["version"] = lv.Version - manifest["time"] = lv.PushedAt.Format(time.RFC3339) - // As for routed packages, keep Composer from reporting installs of - // a private version to the default repository's notify-batch. - // Composer reads a null notification-url as absent too. - if manifest["notification-url"] == nil { - manifest["notification-url"] = "" - } - manifest["dist"] = map[string]string{ - "type": "zip", - "url": h.localDistURL(packageName, lv.Version, lv.Filename), - "reference": lv.SHA256, - "shasum": "", - } - entry, err := json.Marshal(manifest) - if err != nil { - return nil, err - } - entries = append(entries, composerLocalEntry{version: lv.Version, entry: entry}) - } - return entries, nil -} - -func (h *ComposerHandler) localDistURL(packageName, version, filename string) string { - return fmt.Sprintf("%s/composer/files/%s/%s/%s", h.proxyURL, packageName, url.PathEscape(version), filename) -} - -// pushedComposerVersion reports whether version is one of the pushed ones. -// Tags are listed with and without a leading v, and both name one version. -func pushedComposerVersion(local []composerLocalEntry, version string) bool { - for _, e := range local { - if strings.TrimPrefix(e.version, "v") == strings.TrimPrefix(version, "v") { - return true - } - } - return false -} - -// serveMetadataWithLocal serves a package's metadata with its pushed versions -// added, bypassing the rewrite cache like the npm handler does. When the -// upstream does not know the package, the document holds only the pushed -// versions. -func (h *ComposerHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, cacheKey string, local []composerLocalEntry) { - upstream := h.proxy.fetchMetadataDocument(r.Context(), "composer", cacheKey, upstreamURL, contentTypeJSON, nil) - var out []byte - var err error - switch { - case errors.Is(upstream.err, ErrUpstreamNotFound): - out, err = localComposerDocument(packageName, local) - case upstream.err != nil: - if writeUpstreamHeld(w, upstream.err) { - return - } - h.proxy.Logger.Error("upstream request failed", "error", upstream.err) - http.Error(w, "upstream request failed", http.StatusBadGateway) - return - default: - out, _, err = h.rewriteMetadataWithLocal(upstream.body, packageName, local) - } - if err != nil { - h.proxy.Logger.Error("failed to add pushed versions to composer metadata", "package", packageName, "error", err) - http.Error(w, "failed to rewrite package metadata", http.StatusBadGateway) - return - } - markStale(w, upstream.stale) - serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("composer", h.proxyURL, cacheKey, out)) -} - -// serveHostedOnlyMetadata answers a metadata request for a package of a -// hosted-only vendor from its pushed versions alone. -func (h *ComposerHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http.Request, packageName, cacheKey string, local []composerLocalEntry) { - if len(local) == 0 { - http.Error(w, "not found", http.StatusNotFound) - return - } - out, err := localComposerDocument(packageName, local) - if err != nil { - h.proxy.Logger.Error("failed to build composer metadata from pushed versions", "package", packageName, "error", err) - http.Error(w, "failed to build package metadata", http.StatusInternalServerError) - return - } - serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("composer", h.proxyURL, cacheKey, out)) -} - -func localComposerDocument(packageName string, local []composerLocalEntry) ([]byte, error) { - entries := make([]json.RawMessage, 0, len(local)) - for _, e := range local { - entries = append(entries, e.entry) - } - return json.Marshal(map[string]any{"packages": map[string]any{packageName: entries}}) -} diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go deleted file mode 100644 index dd792ca..0000000 --- a/internal/handler/composer_hosted_test.go +++ /dev/null @@ -1,270 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/registries/fetch" -) - -const monologMetadata = `{"minified":"composer/2.0","packages":{"monolog/monolog":[ - {"name":"monolog/monolog","version":"3.5.0","version_normalized":"3.5.0.0", - "require":{"php":">=8.1","psr/log":"^2.0"},"support":{"issues":"https://github.com/Seldaek/monolog/issues"}, - "dist":{"type":"zip","url":"https://api.github.com/repos/Seldaek/monolog/zipball/aaa","reference":"aaa","shasum":""}, - "time":"2023-10-27T15:32:31+00:00"}, - {"version":"3.4.0","version_normalized":"3.4.0.0", - "dist":{"type":"zip","url":"https://api.github.com/repos/Seldaek/monolog/zipball/bbb","reference":"bbb","shasum":""}} -]}}` - -const monologDevMetadata = `{"minified":"composer/2.0","packages":{"monolog/monolog":[ - {"name":"monolog/monolog","version":"dev-main","version_normalized":"dev-main", - "dist":{"type":"zip","url":"https://api.github.com/repos/Seldaek/monolog/zipball/ccc","reference":"ccc","shasum":""}} -]}}` - -type composerHostedFixture struct { - proxy *Proxy - handler http.Handler - upstreamRequests *int - fetcher *mockFetcher -} - -func newComposerHostedFixture(t *testing.T, upstream map[string]string) composerHostedFixture { - t.Helper() - upstreamRequests := 0 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - body, ok := upstream[r.URL.Path] - if !ok { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(body)) - })) - t.Cleanup(srv.Close) - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HostedEnabled = true - proxy.HTTPClient = srv.Client() - h := NewComposerHandlerWithUpstreams(proxy, "http://proxy.local", srv.URL, srv.URL) - return composerHostedFixture{proxy: proxy, handler: h.Routes(), upstreamRequests: &upstreamRequests, fetcher: fetcher} -} - -func (f composerHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { - t.Helper() - target, err := HostedTargetFor("composer", name, version, "") - if err != nil { - t.Fatalf("HostedTargetFor: %v", err) - } - pushed, err := f.proxy.PushLocalArtifact(context.Background(), "composer", target, bytes.NewReader(composerZip("", manifest)), "", "") - if err != nil { - t.Fatalf("PushLocalArtifact: %v", err) - } - return pushed -} - -func (f composerHostedFixture) versions(t *testing.T, path, packageName string) map[string]map[string]any { - t.Helper() - rec := httptest.NewRecorder() - f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) - if rec.Code != http.StatusOK { - t.Fatalf("GET %s status = %d, body %s", path, rec.Code, rec.Body) - } - byVersion := map[string]map[string]any{} - for _, v := range composerVersions(t, rec.Body.Bytes(), packageName) { - version, _ := v["version"].(string) - if _, dup := byVersion[version]; dup { - t.Fatalf("version %s listed twice:\n%s", version, rec.Body) - } - byVersion[version] = v - } - return byVersion -} - -func TestComposerMetadataListsPushedVersion(t *testing.T) { - f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) - pushed := f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog","require":{"php":">=8.1","psr/log":"^3.0"},"repositories":[{"type":"vcs","url":"x"}]}`) - - versions := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") - local, ok := versions["3.6.0"] - if !ok { - t.Fatalf("pushed version missing: %v", versions) - } - if require, _ := local["require"].(map[string]any); require["psr/log"] != "^3.0" { - t.Errorf("require = %v, want the pushed composer.json's", local["require"]) - } - for _, inherited := range []string{"version_normalized", "support", "repositories"} { - if _, ok := local[inherited]; ok { - t.Errorf("pushed version carries %s = %v", inherited, local[inherited]) - } - } - dist, _ := local["dist"].(map[string]any) - if dist["url"] != "http://proxy.local/composer/files/monolog/monolog/3.6.0/monolog-3.6.0.zip" || - dist["type"] != "zip" || dist["reference"] != pushed.SHA256 || dist["shasum"] != "" { - t.Errorf("dist = %v", dist) - } - if local["time"] != pushed.PushedAt.Format(time.RFC3339) { - t.Errorf("time = %v, want %s", local["time"], pushed.PushedAt.Format(time.RFC3339)) - } - if _, ok := versions["3.4.0"]; !ok { - t.Error("upstream version 3.4.0 missing") - } -} - -func TestComposerMetadataPushedVersionReplacesUpstream(t *testing.T) { - f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) - f.push(t, "monolog/monolog", "3.5.0", `{"name":"monolog/monolog","require":{"php":">=8.1","psr/log":"^3.0"}}`) - - versions := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") - local := versions["3.5.0"] - if require, _ := local["require"].(map[string]any); require["psr/log"] != "^3.0" { - t.Errorf("require = %v, want the pushed composer.json's", local["require"]) - } - if dist, _ := local["dist"].(map[string]any); dist["url"] != "http://proxy.local/composer/files/monolog/monolog/3.5.0/monolog-3.5.0.zip" { - t.Errorf("dist = %v", dist) - } - if v34 := versions["3.4.0"]; v34["version_normalized"] != "3.4.0.0" { - t.Errorf("upstream 3.4.0 expanded to %v", v34) - } -} - -func TestComposerMetadataSplitsPushedDevVersions(t *testing.T) { - f := newComposerHostedFixture(t, map[string]string{ - "/p2/monolog/monolog.json": monologMetadata, - "/p2/monolog/monolog~dev.json": monologDevMetadata, - }) - f.push(t, "monolog/monolog", "dev-patched", `{"name":"monolog/monolog"}`) - f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) - - tagged := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") - dev := f.versions(t, "/p2/monolog/monolog~dev.json", "monolog/monolog") - if _, ok := tagged["dev-patched"]; ok { - t.Error("pushed dev version listed with the tagged versions") - } - if _, ok := dev["dev-patched"]; !ok { - t.Errorf("pushed dev version missing from the dev file: %v", dev) - } - if _, ok := dev["3.6.0"]; ok { - t.Error("pushed tagged version listed with the dev versions") - } - if _, ok := dev["dev-main"]; !ok { - t.Error("upstream dev-main missing") - } -} - -func TestComposerMetadataForPackageOnlyPushed(t *testing.T) { - f := newComposerHostedFixture(t, nil) - f.push(t, "acme/private", "1.0.0", `{"name":"acme/private","require":{"php":">=8.1"}}`) - - versions := f.versions(t, "/p2/acme/private.json", "acme/private") - if _, ok := versions["1.0.0"]; !ok || len(versions) != 1 { - t.Errorf("versions = %v, want only 1.0.0", versions) - } - - rec := httptest.NewRecorder() - f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/p2/acme/private~dev.json", nil)) - if rec.Code != http.StatusNotFound { - t.Errorf("dev file of a package with no pushed dev versions status = %d, want 404", rec.Code) - } -} - -func TestComposerHostedOnlyVendorNeverAsksUpstream(t *testing.T) { - squatter := `{"packages":{"acme/private":[{"name":"acme/private","version":"1.0.1", - "dist":{"type":"zip","url":"https://api.github.com/repos/evil/private/zipball/ddd","reference":"ddd"}}]}}` - f := newComposerHostedFixture(t, map[string]string{ - "/p2/acme/private.json": squatter, - "/p2/monolog/monolog.json": monologMetadata, - }) - f.proxy.HostedComposerVendors = []string{"acme"} - f.push(t, "acme/private", "1.0.0", `{"name":"acme/private"}`) - - versions := f.versions(t, "/p2/acme/private.json", "acme/private") - if _, ok := versions["1.0.0"]; !ok || len(versions) != 1 { - t.Errorf("versions = %v, want only the pushed 1.0.0", versions) - } - for _, path := range []string{"/p2/acme/private~dev.json", "/p2/acme/other.json", "/files/acme/private/1.0.1/ddd.zip"} { - rec := httptest.NewRecorder() - f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) - if rec.Code != http.StatusNotFound { - t.Errorf("GET %s status = %d, want 404", path, rec.Code) - } - } - if *f.upstreamRequests != 0 { - t.Errorf("hosted-only vendor sent %d requests upstream", *f.upstreamRequests) - } - - f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") - if *f.upstreamRequests == 0 { - t.Error("package outside the vendor was not fetched upstream") - } -} - -func TestComposerDownloadOfUnreadablePushedArchive(t *testing.T) { - f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) - f.push(t, "monolog/monolog", "3.5.0", `{"name":"monolog/monolog"}`) - removeStored(t, f.proxy, "pkg:composer/monolog/monolog@3.5.0", "monolog-3.5.0.zip") - - rec := httptest.NewRecorder() - f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/files/monolog/monolog/3.5.0/monolog-3.5.0.zip", nil)) - if rec.Code != http.StatusServiceUnavailable { - t.Errorf("download status = %d, body %s, want 503 rather than the upstream archive", rec.Code, rec.Body) - } -} - -func TestComposerMetadataLeavesOutDeniedPushedVersions(t *testing.T) { - f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) - f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) - setTestDenylist(t, f.proxy, "pkg:composer/monolog/monolog@3.6.0") - - if _, ok := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog")["3.6.0"]; ok { - t.Error("denied pushed version listed") - } -} - -func TestComposerDownloadOfPushedArchiveWhileStreamingArtifacts(t *testing.T) { - const manifest = `{"name":"monolog/monolog"}` - f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) - f.fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream archive")), ContentType: "application/zip"} - f.push(t, "monolog/monolog", "3.5.0", manifest) - f.proxy.StreamArtifacts = true - - rec := httptest.NewRecorder() - f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/files/monolog/monolog/3.5.0/monolog-3.5.0.zip", nil)) - if rec.Code != http.StatusOK || rec.Body.String() != string(composerZip("", manifest)) { - t.Errorf("download status = %d, want 200 and the pushed archive", rec.Code) - } - if f.fetcher.fetchCalled { - t.Error("a pushed archive was fetched upstream while streaming artifacts") - } -} - -func TestComposerPushedVersionsDisableDefaultNotification(t *testing.T) { - f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) - f.proxy.HostedComposerVendors = []string{"acme"} - f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) - f.push(t, "acme/private", "1.0.0", `{"name":"acme/private"}`) - f.push(t, "other/private", "1.0.0", `{"name":"other/private"}`) - // Composer reads a null notification-url as absent. - f.push(t, "acme/nulled", "1.0.0", `{"name":"acme/nulled","notification-url":null}`) - f.push(t, "acme/own", "1.0.0", `{"name":"acme/own","notification-url":"https://acme.example.com/notify"}`) - - for _, tc := range []struct{ path, name, version, want string }{ - {"/p2/monolog/monolog.json", "monolog/monolog", "3.6.0", ""}, - {"/p2/acme/private.json", "acme/private", "1.0.0", ""}, - {"/p2/other/private.json", "other/private", "1.0.0", ""}, - {"/p2/acme/nulled.json", "acme/nulled", "1.0.0", ""}, - // A manifest's own URL is kept. - {"/p2/acme/own.json", "acme/own", "1.0.0", "https://acme.example.com/notify"}, - } { - v := f.versions(t, tc.path, tc.name)[tc.version] - if got, ok := v["notification-url"]; !ok || got != tc.want { - t.Errorf("%s %s notification-url = %v (set %v), want %q", tc.name, tc.version, got, ok, tc.want) - } - } -} diff --git a/internal/handler/composer_rewrite_bench_test.go b/internal/handler/composer_rewrite_bench_test.go deleted file mode 100644 index 075b371..0000000 --- a/internal/handler/composer_rewrite_bench_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -// benchComposerMetadata builds minified Composer v2 metadata shaped like a -// large real package: thousands of versions, where the first carries every -// field and the rest mostly change version, time, dist and source, with the -// requirements changing every so often. -func benchComposerMetadata(versions int) []byte { - var b strings.Builder - b.WriteString(`{"minified":"composer/2.0","packages":{"big/sdk":[`) - for i := range versions { - if i > 0 { - b.WriteByte(',') - } - v := fmt.Sprintf("3.%d.0", versions-i) - ref := fmt.Sprintf("%040x", i) - b.WriteByte('{') - if i == 0 { - b.WriteString(`"name":"big/sdk","description":"A big SDK for a big cloud",` + - `"keywords":["cloud","sdk","api","storage","queue"],"homepage":"https://example.com/sdk",` + - `"license":["Apache-2.0"],"authors":[{"name":"Big Cloud","homepage":"https://example.com"}],` + - `"type":"library","autoload":{"psr-4":{"Big\\Sdk\\":"src/"},"files":["src/functions.php"]},` + - `"support":{"issues":"https://example.com/sdk/issues","source":"https://example.com/sdk/tree/main"},` + - `"extra":{"branch-alias":{"dev-master":"3.0-dev"}},`) - } - if i%50 == 0 { - b.WriteString(`"require":{"php":">=8.1",`) - for d := range 12 { - if d > 0 { - b.WriteByte(',') - } - fmt.Fprintf(&b, `"vendor/dep-%d":"^%d.%d"`, d, d+1, i/50) - } - b.WriteString(`},`) - } - fmt.Fprintf(&b, `"version":%q,"version_normalized":"%s.0","time":"2024-01-01T00:00:00+00:00",`+ - `"source":{"url":"https://github.com/big/sdk.git","type":"git","reference":%q},`+ - `"dist":{"url":"https://api.github.com/repos/big/sdk/zipball/%s","type":"zip","shasum":"","reference":%q}}`, - v, v, ref, ref, ref) - } - b.WriteString(`]}}`) - return []byte(b.String()) -} - -func BenchmarkComposerRewriteMetadata(b *testing.B) { - body := benchComposerMetadata(5000) - h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy.example"} - b.SetBytes(int64(len(body))) - b.ReportAllocs() - for b.Loop() { - if _, _, err := h.rewriteMetadataKeeping(body); err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkComposerFindDownloadURL(b *testing.B) { - body := benchComposerMetadata(5000) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write(body) - })) - defer srv.Close() - h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy.example"} - h.proxy.HTTPClient = srv.Client() - b.SetBytes(int64(len(body))) - b.ReportAllocs() - for b.Loop() { - dist, err := h.findDownloadURLFromMetadata(context.Background(), srv.URL, "big/sdk", "3.2500.0") - if err != nil || dist.url == "" { - b.Fatalf("download URL not found: %q, %v", dist.url, err) - } - } -} diff --git a/internal/handler/composer_rewrite_test.go b/internal/handler/composer_rewrite_test.go deleted file mode 100644 index 460e38e..0000000 --- a/internal/handler/composer_rewrite_test.go +++ /dev/null @@ -1,282 +0,0 @@ -package handler - -import ( - "encoding/json" - "log/slog" - "maps" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// composerVersions decodes rewritten metadata and returns packageName's -// versions as a Composer client sees them: minified lists are expanded the -// way Composer's MetadataMinifier::expand does it. -func composerVersions(t *testing.T, output []byte, packageName string) []map[string]any { - t.Helper() - var doc struct { - Minified string `json:"minified"` - Packages map[string]any `json:"packages"` - } - if err := json.Unmarshal(output, &doc); err != nil { - t.Fatalf("output is not JSON: %v\n%s", err, output) - } - var versions []map[string]any - var expanded map[string]any - entries, _ := doc.Packages[packageName].([]any) - for _, entry := range entries { - if doc.Minified == "composer/2.0" && entry == "~dev" { - expanded = nil - continue - } - fields, ok := entry.(map[string]any) - if !ok { - t.Fatalf("version entry is not an object: %v", entry) - } - if doc.Minified != "composer/2.0" || expanded == nil { - expanded = fields - versions = append(versions, fields) - continue - } - expanded = maps.Clone(expanded) - for k, v := range fields { - if v == composerUnset { - delete(expanded, k) - continue - } - expanded[k] = v - } - versions = append(versions, expanded) - } - return versions -} - -func versionNames(versions []map[string]any) []string { - names := make([]string, len(versions)) - for i, v := range versions { - names[i], _ = v["version"].(string) - } - return names -} - -func distURL(t *testing.T, version map[string]any) string { - t.Helper() - dist, ok := version["dist"].(map[string]any) - if !ok { - t.Fatalf("version %v has no dist object", version["version"]) - } - url, _ := dist["url"].(string) - return url -} - -func cooldownComposerHandler() *ComposerHandler { - proxy := &Proxy{Logger: slog.Default()} - proxy.Cooldown = &cooldown.Config{Default: "3d"} - return &ComposerHandler{proxy: proxy, proxyURL: "http://proxy"} -} - -// TestComposerRewriteMetadataKeepsUpstreamBytes checks that only dist URLs -// change: numbers encoding/json would round through float64, characters it -// would HTML-escape and the key order all reach the client as upstream sent -// them. -func TestComposerRewriteMetadataKeepsUpstreamBytes(t *testing.T) { - h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} - input := `{"packages":{"v/p":[{"version":"1.0.0","extra":{"big":12345678901234567890,"html":"&"},` + - `"dist":{"type":"zip","url":"https://example.com/abc.zip","reference":"abc"}}]},"security-advisories":[]}` - - out, _, err := h.rewriteMetadataKeeping([]byte(input)) - if err != nil { - t.Fatal(err) - } - want := strings.Replace(input, "https://example.com/abc.zip", "http://proxy/composer/files/v/p/1.0.0/abc.zip", 1) - if string(out) != want { - t.Errorf("rewrite =\n%s\nwant\n%s", out, want) - } -} - -// TestComposerRewriteMetadataStaysMinified checks that minified metadata is -// rewritten without expanding it: each entry still carries only what changed -// from the one before, plus its own dist now that dist URLs name the version. -func TestComposerRewriteMetadataStaysMinified(t *testing.T) { - h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} - input := `{"minified":"composer/2.0","packages":{"v/p":[` + - `{"name":"v/p","version":"2.0.0","require":{"php": ">=8.1"},"dist":{"url":"https://example.com/a.zip","type":"zip"}},` + - `{"version":"1.1.0","dist":{"url":"https://example.com/b.zip","type":"zip"}},` + - `{"version":"1.0.0","require":"__unset"}` + - `]}}` - - out, _, err := h.rewriteMetadataKeeping([]byte(input)) - if err != nil { - t.Fatal(err) - } - want := `{"minified":"composer/2.0","packages":{"v/p":[` + - `{"name":"v/p","version":"2.0.0","require":{"php": ">=8.1"},"dist":{"url":"http://proxy/composer/files/v/p/2.0.0/a.zip","type":"zip"}},` + - `{"version":"1.1.0","dist":{"url":"http://proxy/composer/files/v/p/1.1.0/b.zip","type":"zip"}},` + - `{"version":"1.0.0","dist":{"url":"http://proxy/composer/files/v/p/1.0.0/b.zip","type":"zip"},"require":"__unset"}` + - `]}}` - if string(out) != want { - t.Errorf("rewrite =\n%s\nwant\n%s", out, want) - } -} - -// TestComposerRewriteMetadataCooldownCarriesChanges checks that the changes a -// filtered entry made to the inherited fields still reach the next entry -// that is kept. -func TestComposerRewriteMetadataCooldownCarriesChanges(t *testing.T) { - old := time.Now().Add(-10 * 24 * time.Hour).Format(time.RFC3339) - recent := time.Now().Add(-1 * time.Hour).Format(time.RFC3339) - input := `{"minified":"composer/2.0","packages":{"v/p":[ - {"name":"v/p","description":"Seven","version":"7.0.0","time":"` + recent + `", - "require-dev":{"phpunit/phpunit":"^11"},"dist":{"url":"https://example.com/7.zip","type":"zip"}}, - {"version":"6.4.0","time":"` + old + `","dist":{"url":"https://example.com/640.zip","type":"zip"}}, - {"description":"Backport","version":"5.4.9","time":"` + recent + `","require-dev":"__unset", - "dist":{"url":"https://example.com/549.zip","type":"zip"}}, - {"version":"5.4.8","time":"` + old + `"} - ]}}` - - out, _, err := cooldownComposerHandler().rewriteMetadataKeeping([]byte(input)) - if err != nil { - t.Fatal(err) - } - versions := composerVersions(t, out, "v/p") - if got := versionNames(versions); strings.Join(got, ",") != "6.4.0,5.4.8" { - t.Fatalf("versions = %v, want [6.4.0 5.4.8]", got) - } - - v640, v548 := versions[0], versions[1] - if v640["name"] != "v/p" || v640["description"] != "Seven" || v640["require-dev"] == nil { - t.Errorf("6.4.0 lost fields inherited from the filtered 7.0.0: %v", v640) - } - if v548["description"] != "Backport" { - t.Errorf("5.4.8 description = %v, want the one set by the filtered 5.4.9", v548["description"]) - } - if _, ok := v548["require-dev"]; ok { - t.Errorf("5.4.8 require-dev = %v, want it unset by the filtered 5.4.9", v548["require-dev"]) - } - if got := distURL(t, v548); got != "http://proxy/composer/files/v/p/5.4.8/549.zip" { - t.Errorf("5.4.8 dist url = %q", got) - } -} - -// TestComposerRewriteMetadataCooldownAfterDevReset checks that a "~dev" -// reset survives when the first dev entry after it is filtered. -func TestComposerRewriteMetadataCooldownAfterDevReset(t *testing.T) { - old := time.Now().Add(-10 * 24 * time.Hour).Format(time.RFC3339) - recent := time.Now().Add(-1 * time.Hour).Format(time.RFC3339) - input := `{"minified":"composer/2.0","packages":{"v/p":[ - {"name":"v/p","license":["MIT"],"version":"1.0.0","time":"` + old + `", - "dist":{"url":"https://example.com/1.zip","type":"zip"}}, - "~dev", - {"name":"v/p","description":"Dev","version":"dev-main","time":"` + recent + `", - "dist":{"url":"https://example.com/main.zip","type":"zip"}}, - {"version":"1.x-dev","time":"` + old + `"} - ]}}` - - out, _, err := cooldownComposerHandler().rewriteMetadataKeeping([]byte(input)) - if err != nil { - t.Fatal(err) - } - versions := composerVersions(t, out, "v/p") - if got := versionNames(versions); strings.Join(got, ",") != "1.0.0,1.x-dev" { - t.Fatalf("versions = %v, want [1.0.0 1.x-dev]", got) - } - dev := versions[1] - if _, ok := dev["license"]; ok { - t.Error("1.x-dev inherited license across the ~dev reset") - } - if dev["name"] != "v/p" || dev["description"] != "Dev" { - t.Errorf("1.x-dev lost fields inherited from the filtered dev-main: %v", dev) - } - if got := distURL(t, dev); got != "http://proxy/composer/files/v/p/1.x-dev/main.zip" { - t.Errorf("1.x-dev dist url = %q", got) - } -} - -func TestComposerRewriteMetadataInvalidJSON(t *testing.T) { - h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} - if _, _, err := h.rewriteMetadataKeeping([]byte(`{"packages":{"v/p":[{"version":"1.0.0"}]}`)); err == nil { - t.Error("expected an error for truncated JSON") - } -} - -func TestComposerRewriteMetadataWithoutPackages(t *testing.T) { - h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} - for _, input := range []string{`{"other":1}`, `{"packages":[]}`} { - out, _, err := h.rewriteMetadataKeeping([]byte(input)) - if err != nil { - t.Fatalf("%s: %v", input, err) - } - if string(out) != input { - t.Errorf("rewrite of %s = %s, want it unchanged", input, out) - } - } -} - -func TestComposerDistURL(t *testing.T) { - minified := `{"minified":"composer/2.0","packages":{"v/p":[ - {"name":"v/p","version":"3.0.0","dist":{"url":"https://example.com/3.zip","type":"zip"}}, - {"version":"2.0.0"}, - {"version":"1.0.0","dist":"__unset"}, - "~dev", - {"version":"dev-main","dist":{"url":"https://example.com/main.zip","type":"zip"}} - ]}}` - expanded := `{"packages":{"v/p":[ - {"version":"2.0.0","dist":{"url":"https://example.com/2.zip","type":"zip"}}, - {"version":"1.0.0"} - ]}}` - - tests := []struct { - name, body, version, want string - }{ - {"own dist", minified, "3.0.0", "https://example.com/3.zip"}, - {"inherited dist", minified, "2.0.0", "https://example.com/3.zip"}, - {"unset dist", minified, "1.0.0", ""}, - {"after dev reset", minified, "dev-main", "https://example.com/main.zip"}, - {"missing version", minified, "9.9.9", ""}, - {"expanded", expanded, "2.0.0", "https://example.com/2.zip"}, - {"expanded inherits nothing", expanded, "1.0.0", ""}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := composerDistURL([]byte(tt.body), "v/p", tt.version) - if err != nil { - t.Fatal(err) - } - if got.url != tt.want { - t.Errorf("composerDistURL(%s) = %q, want %q", tt.version, got.url, tt.want) - } - }) - } -} - -func TestComposerDistURLMalformed(t *testing.T) { - if _, err := composerDistURL([]byte(`{"packages":{"v/p":[{"version":`), "v/p", "1.0.0"); err == nil { - t.Error("expected an error for truncated JSON") - } -} - -// TestComposerRewriteMetadataRewritesEveryPackage checks that each package -// list in the document is rewritten under its own name, as before. -func TestComposerRewriteMetadataRewritesEveryPackage(t *testing.T) { - h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} - input := `{"packages":{ - "v/a":[{"version":"1.0.0","dist":{"url":"https://example.com/a.zip","type":"zip"}}], - "v/b":[{"version":"2.0.0","dist":{"url":"https://example.com/b.zip","type":"zip"}}], - "v/c":"not a list" - }}` - out, _, err := h.rewriteMetadataKeeping([]byte(input)) - if err != nil { - t.Fatal(err) - } - if got := distURL(t, composerVersions(t, out, "v/a")[0]); got != "http://proxy/composer/files/v/a/1.0.0/a.zip" { - t.Errorf("v/a dist url = %q", got) - } - if got := distURL(t, composerVersions(t, out, "v/b")[0]); got != "http://proxy/composer/files/v/b/2.0.0/b.zip" { - t.Errorf("v/b dist url = %q", got) - } - if !strings.Contains(string(out), `"v/c":"not a list"`) { - t.Errorf("non-list package value was not copied unchanged:\n%s", out) - } -} diff --git a/internal/handler/composer_routes_test.go b/internal/handler/composer_routes_test.go deleted file mode 100644 index 0ebf5ac..0000000 --- a/internal/handler/composer_routes_test.go +++ /dev/null @@ -1,307 +0,0 @@ -package handler - -import ( - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - "time" - - "github.com/git-pkgs/registries/fetch" -) - -// composerUpstreamRecorder serves canned Composer metadata and records which -// paths were requested. -type composerUpstreamRecorder struct { - *httptest.Server - mu sync.Mutex - requests []string -} - -func newComposerUpstream(t *testing.T, documents map[string]string, status int) *composerUpstreamRecorder { - t.Helper() - u := &composerUpstreamRecorder{} - u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - u.mu.Lock() - u.requests = append(u.requests, r.URL.Path) - u.mu.Unlock() - if status != http.StatusOK { - w.WriteHeader(status) - return - } - body, ok := documents[r.URL.Path] - if !ok { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, body) - })) - t.Cleanup(u.Close) - return u -} - -func (u *composerUpstreamRecorder) requested() []string { - u.mu.Lock() - defer u.mu.Unlock() - return append([]string(nil), u.requests...) -} - -const routedComposerMetadata = `{ - "packages": { - "example/library": [ - {"name": "example/library", "version": "1.0.0", "dist": {"url": "https://private.example.com/archives/library-1.0.0.zip", "type": "zip"}} - ] - } -}` - -const publicComposerMetadata = `{ - "packages": { - "example/library": [ - {"name": "example/library", "version": "9.9.9", "dist": {"url": "https://public.example.com/library-9.9.9.zip", "type": "zip"}} - ] - } -}` - -func newRoutedComposerHandler(t *testing.T, proxy *Proxy, public, private *composerUpstreamRecorder) *ComposerHandler { - t.Helper() - return NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL). - WithPackageRoutes(map[string]string{"example/*": private.URL + "/group/packages/composer"}) -} - -func serveComposer(h *ComposerHandler, target string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - return w -} - -func TestComposerRoutedMetadataComesFromRouteOnly(t *testing.T) { - public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) - private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - h := newRoutedComposerHandler(t, proxy, public, private) - - w := serveComposer(h, "/p2/example/library.json") - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if got := public.requested(); len(got) != 0 { - t.Errorf("default repository was queried for a routed package: %v", got) - } - - var metadata map[string]any - if err := json.Unmarshal(w.Body.Bytes(), &metadata); err != nil { - t.Fatalf("parsing response: %v", err) - } - versions := metadata["packages"].(map[string]any)["example/library"].([]any) - version := versions[0].(map[string]any) - if version["version"] != "1.0.0" { - t.Errorf("version = %v, want the routed upstream's 1.0.0", version["version"]) - } - wantDist := "http://proxy.test/composer/files/example/library/1.0.0/library-1.0.0.zip" - if dist := version["dist"].(map[string]any)["url"]; dist != wantDist { - t.Errorf("dist url = %v, want %q", dist, wantDist) - } - if notification, ok := version["notification-url"]; !ok || notification != "" { - t.Errorf("notification-url = %v (present %t), want empty so Composer does not report routed installs to notify-batch", notification, ok) - } -} - -func TestComposerRoutedMetadataDoesNotFallBack(t *testing.T) { - tests := []struct { - name string - privateStatus int - wantStatus int - }{ - {"package missing on route", http.StatusOK, http.StatusNotFound}, - {"route rejects credentials", http.StatusUnauthorized, http.StatusBadGateway}, - {"route unavailable", http.StatusInternalServerError, http.StatusBadGateway}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) - private := newComposerUpstream(t, map[string]string{}, tt.privateStatus) - proxy, _, _, _ := setupTestProxy(t) - h := newRoutedComposerHandler(t, proxy, public, private) - - w := serveComposer(h, "/p2/example/library.json") - - if w.Code != tt.wantStatus { - t.Errorf("status = %d, want %d", w.Code, tt.wantStatus) - } - if got := public.requested(); len(got) != 0 { - t.Errorf("default repository was queried for a routed package: %v", got) - } - }) - } -} - -func TestComposerRoutedMetadataIgnoresDefaultRepositoryCache(t *testing.T) { - public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) - private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - - unrouted := NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL) - if w := serveComposer(unrouted, "/p2/example/library.json"); w.Code != http.StatusOK { - t.Fatalf("priming the default repository cache: status %d", w.Code) - } - - w := serveComposer(newRoutedComposerHandler(t, proxy, public, private), "/p2/example/library.json") - - if strings.Contains(w.Body.String(), "9.9.9") { - t.Errorf("routed package served metadata cached from the default repository: %s", w.Body.String()) - } - if len(private.requested()) == 0 { - t.Error("routed upstream was not queried") - } -} - -func TestComposerUnroutedMetadataUsesDefaultRepository(t *testing.T) { - public := newComposerUpstream(t, map[string]string{"/p2/other/library.json": strings.ReplaceAll(publicComposerMetadata, "example/", "other/")}, http.StatusOK) - private := newComposerUpstream(t, map[string]string{}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - h := newRoutedComposerHandler(t, proxy, public, private) - - w := serveComposer(h, "/p2/other/library.json") - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) - } - if got := private.requested(); len(got) != 0 { - t.Errorf("route was queried for an unrouted package: %v", got) - } - if strings.Contains(w.Body.String(), "notification-url") { - t.Errorf("unrouted package metadata gained a notification-url: %s", w.Body.String()) - } -} - -func TestComposerRoutedDownloadResolvesFromRoute(t *testing.T) { - public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) - private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) - proxy, _, _, artifactFetcher := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("archive")), - ContentType: "application/zip", - } - h := newRoutedComposerHandler(t, proxy, public, private) - - w := serveComposer(h, "/files/example/library/1.0.0/library-1.0.0.zip") - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if want := "https://private.example.com/archives/library-1.0.0.zip"; artifactFetcher.fetchedURL != want { - t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) - } - if got := public.requested(); len(got) != 0 { - t.Errorf("default repository was queried for a routed download: %v", got) - } -} - -func TestComposerRoutedDownloadIgnoresDefaultRepositoryArtifact(t *testing.T) { - public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": strings.ReplaceAll(publicComposerMetadata, "9.9.9", "1.0.0")}, http.StatusOK) - private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) - proxy, _, _, artifactFetcher := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("public archive")), - ContentType: "application/zip", - } - - unrouted := NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL) - if w := serveComposer(unrouted, "/files/example/library/1.0.0/library-1.0.0.zip"); w.Code != http.StatusOK { - t.Fatalf("priming the artifact cache: status %d; body: %s", w.Code, w.Body.String()) - } - - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("private archive")), - ContentType: "application/zip", - } - w := serveComposer(newRoutedComposerHandler(t, proxy, public, private), "/files/example/library/1.0.0/library-1.0.0.zip") - - if w.Body.String() != "private archive" { - t.Errorf("routed download served %q, want the routed upstream's archive", w.Body.String()) - } -} - -// TestComposerRoutedMinifiedMetadataDisablesNotification checks that every -// version of a routed package, as Composer expands minified metadata, has an -// empty notification-url: versions that inherit their fields, versions after a -// ~dev reset, versions after one that set its own URL and then unset it, and -// versions with a null URL. -func TestComposerRoutedMinifiedMetadataDisablesNotification(t *testing.T) { - minified := `{"minified":"composer/2.0","packages":{"example/library":[` + - `{"name":"example/library","version":"1.0.0","dist":{"url":"https://private.example.com/a/1.0.0.zip","type":"zip"}},` + - `{"version":"1.1.0","dist":{"url":"https://private.example.com/a/1.1.0.zip","type":"zip"}},` + - `{"version":"1.2.0","notification-url":"https://private.example.com/notify","dist":{"url":"https://private.example.com/a/1.2.0.zip","type":"zip"}},` + - `{"version":"1.3.0","notification-url":"__unset","dist":{"url":"https://private.example.com/a/1.3.0.zip","type":"zip"}},` + - `{"version":"1.4.0","notification-url":null,"dist":{"url":"https://private.example.com/a/1.4.0.zip","type":"zip"}},` + - `{"version":"1.5.0","dist":{"url":"https://private.example.com/a/1.5.0.zip","type":"zip"}},` + - `"~dev",` + - `{"name":"example/library","version":"dev-main","dist":{"url":"https://private.example.com/a/main.zip","type":"zip"}}]}}` - public := newComposerUpstream(t, map[string]string{}, http.StatusOK) - private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": minified}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - h := newRoutedComposerHandler(t, proxy, public, private) - - w := serveComposer(h, "/p2/example/library.json") - if w.Code != http.StatusOK { - t.Fatalf("status = %d; body: %s", w.Code, w.Body.String()) - } - want := map[string]string{ - "1.0.0": "", - "1.1.0": "", - "1.2.0": "https://private.example.com/notify", // the route's own URL is kept - "1.3.0": "", - "1.4.0": "", // Composer reads null as absent - "1.5.0": "", - "dev-main": "", - } - versions := composerVersions(t, w.Body.Bytes(), "example/library") - if len(versions) != len(want) { - t.Fatalf("versions = %v, want %d", versionNames(versions), len(want)) - } - for _, v := range versions { - name, _ := v["version"].(string) - if got, ok := v["notification-url"]; !ok || got != want[name] { - t.Errorf("%s: notification-url = %v (present %t), want %q", name, got, ok, want[name]) - } - } -} - -// TestComposerExactRouteCoversDevMetadata is a regression test: Packagist -// serves a package's dev versions from a separate {name}~dev.json file, and an -// exact route for the package must cover that file too. Matching the route -// against the name with ~dev still attached sent the private package's dev -// metadata to the default repository. -func TestComposerExactRouteCoversDevMetadata(t *testing.T) { - const devMetadata = `{"packages":{"example/library":[{"name":"example/library","version":"dev-main","dist":{"url":"https://private.example.com/archives/main.zip","type":"zip"}}]}}` - public := newComposerUpstream(t, map[string]string{"/p2/example/library~dev.json": publicComposerMetadata}, http.StatusOK) - private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library~dev.json": devMetadata}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - h := NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL). - WithPackageRoutes(map[string]string{"example/library": private.URL + "/group/packages/composer"}) - - w := serveComposer(h, "/p2/example/library~dev.json") - - if w.Code != http.StatusOK { - t.Fatalf("status = %d; body: %s", w.Code, w.Body.String()) - } - if got := public.requested(); len(got) != 0 { - t.Errorf("default repository was queried for a routed package's dev metadata: %v", got) - } - if got, want := private.requested(), []string{"/group/packages/composer/p2/example/library~dev.json"}; strings.Join(got, ",") != strings.Join(want, ",") { - t.Errorf("route requests = %v, want %v", got, want) - } - if !strings.Contains(w.Body.String(), "/composer/files/example/library/dev-main/") { - t.Errorf("dev metadata was not rewritten through the proxy: %s", w.Body.String()) - } -} diff --git a/internal/handler/composer_shasum_test.go b/internal/handler/composer_shasum_test.go deleted file mode 100644 index b484795..0000000 --- a/internal/handler/composer_shasum_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "io" - "net/http" - "net/http/httptest" - "sync" - "testing" - - "github.com/git-pkgs/registries/fetch" -) - -func TestComposerDownloadVerifiesDeclaredShasum(t *testing.T) { - const archive = "archive bytes" - - tests := []struct { - name string - shasum string - wantStatus int - }{ - {"matching shasum", sha1Hex(archive), http.StatusOK}, - {"mismatched shasum", sha1Hex("other bytes"), http.StatusBadGateway}, - {"empty shasum", "", http.StatusOK}, - {"unparseable shasum", "not-a-sha1", http.StatusOK}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - h, proxy := setupComposerShasum(t, "1.0.0", "/zip/p.zip", tt.shasum, map[string]string{"/zip/p.zip": archive}) - - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/1.0.0/p.zip", nil)) - if w.Code != tt.wantStatus { - t.Fatalf("status = %d, want %d; body: %s", w.Code, tt.wantStatus, w.Body.String()) - } - if wantCached := tt.wantStatus == http.StatusOK; isComposerArchiveCached(t, proxy, "1.0.0", "p.zip") != wantCached { - t.Errorf("cached = %v, want %v", !wantCached, wantCached) - } - }) - } -} - -// TestComposerDownloadPinnedCommitIgnoresListedShasum checks that the shasum -// listed for a branch's head is not held against an older commit a lock file -// pins: it describes another archive. -func TestComposerDownloadPinnedCommitIgnoresListedShasum(t *testing.T) { - h, _ := setupComposerShasum(t, "dev-master", "/zip/"+composerCommitB, sha1Hex("commit B"), map[string]string{ - "/zip/" + composerCommitA: "commit A", - "/zip/" + composerCommitB: "commit B", - }) - - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+composerCommitA+".zip", nil)) - if w.Code != http.StatusOK || w.Body.String() != "commit A" { - t.Errorf("status %d, body %q, want 200 %q", w.Code, w.Body.String(), "commit A") - } -} - -func setupComposerShasum(t *testing.T, version, distPath, shasum string, archives map[string]string) (http.Handler, *Proxy) { - t.Helper() - h, proxy, _ := setupComposerShasumCounting(t, version, distPath, shasum, archives) - return h, proxy -} - -// setupComposerShasumCounting is setupComposerShasum that also reports how -// many times each archive path was fetched from upstream. -func setupComposerShasumCounting(t *testing.T, version, distPath, shasum string, archives map[string]string) (http.Handler, *Proxy, func(string) int) { - t.Helper() - var mu sync.Mutex - fetched := map[string]int{} - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/p2/v/p.json" || r.URL.Path == "/p2/v/p~dev.json" { - _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[{"version":%q,"dist":{"url":%q,"type":"zip","shasum":%q}}]}}`, - version, srv.URL+distPath, shasum) - return - } - body, ok := archives[r.URL.Path] - if !ok { - http.NotFound(w, r) - return - } - mu.Lock() - fetched[r.URL.Path]++ - mu.Unlock() - _, _ = io.WriteString(w, body) - })) - t.Cleanup(srv.Close) - - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - return NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes(), proxy, func(path string) int { - mu.Lock() - defer mu.Unlock() - return fetched[path] - } -} - -func isComposerArchiveCached(t *testing.T, proxy *Proxy, version, filename string) bool { - t.Helper() - cached, err := proxy.GetCachedArtifact(context.Background(), "composer", "v/p", version, filename) - if err != nil { - t.Fatalf("reading cache: %v", err) - } - if cached == nil { - return false - } - _ = cached.Reader.Close() - return true -} - -// TestComposerDevBranchArchivesStayServableFromCache is a regression test: a -// version's integrity is one value shared by all its cached archives, but a -// dev branch caches one archive per commit. Recording the head's shasum made -// a pinned older commit fail its check when served from the cache, so its -// archive was discarded and fetched again, which in turn cleared the head's -// recorded shasum. -func TestComposerDevBranchArchivesStayServableFromCache(t *testing.T) { - h, proxy, fetched := setupComposerShasumCounting(t, "dev-master", "/zip/"+composerCommitB, sha1Hex("commit B"), map[string]string{ - "/zip/" + composerCommitA: "commit A", - "/zip/" + composerCommitB: "commit B", - }) - - for i, commit := range []string{composerCommitA, composerCommitB, composerCommitA, composerCommitB} { - want := map[string]string{composerCommitA: "commit A", composerCommitB: "commit B"}[commit] - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+commit+".zip", nil)) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Errorf("request %d for %s: status %d, body %q; want 200 %q", i+1, commit[:4], w.Code, w.Body.String(), want) - } - } - for _, commit := range []string{composerCommitA, composerCommitB} { - if n := fetched("/zip/" + commit); n != 1 { - t.Errorf("commit %s fetched from upstream %d times, want once and then served from the cache", commit[:4], n) - } - if !isComposerArchiveCached(t, proxy, "dev-master", commit+".zip") { - t.Errorf("commit %s is no longer cached", commit[:4]) - } - } -} diff --git a/internal/handler/composer_test.go b/internal/handler/composer_test.go index a964dfc..9898664 100644 --- a/internal/handler/composer_test.go +++ b/internal/handler/composer_test.go @@ -3,19 +3,14 @@ package handler import ( "context" "encoding/json" - "fmt" - "io" "log/slog" "net/http" "net/http/httptest" "strings" - "sync/atomic" "testing" "time" "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries/fetch" ) func TestComposerRewriteMetadata(t *testing.T) { @@ -38,7 +33,7 @@ func TestComposerRewriteMetadata(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping([]byte(input)) + output, err := h.rewriteMetadata([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -59,16 +54,15 @@ func TestComposerRewriteMetadata(t *testing.T) { } } -func TestComposerRewriteMetadataMinifiedInheritance(t *testing.T) { +func TestComposerRewriteMetadataExpandsMinified(t *testing.T) { h := &ComposerHandler{ proxy: testProxy(), proxyURL: "http://localhost:8080", } // Minified format: first version has all fields, subsequent versions - // only include fields that changed. The proxy keeps the format, so once - // Composer expands it every version still has all fields (including - // "name"). + // only include fields that changed. The proxy must expand this so every + // version has all fields (including "name"). input := `{ "minified": "composer/2.0", "packages": { @@ -93,7 +87,7 @@ func TestComposerRewriteMetadataMinifiedInheritance(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping([]byte(input)) + output, err := h.rewriteMetadata([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -103,14 +97,16 @@ func TestComposerRewriteMetadataMinifiedInheritance(t *testing.T) { t.Fatalf("failed to parse output: %v", err) } - if result["minified"] != "composer/2.0" { - t.Errorf("minified = %v, want the format kept", result["minified"]) + // The minified key should be removed from output + if _, ok := result["minified"]; ok { + t.Error("expected minified key to be removed from output") } - versions := composerVersions(t, output, "symfony/console") + packages := result["packages"].(map[string]any) + versions := packages["symfony/console"].([]any) // Second version should have inherited the "name" and "description" fields - v1 := versions[1] + v1 := versions[1].(map[string]any) if v1["name"] != "symfony/console" { t.Errorf("second version name = %v, want %q", v1["name"], "symfony/console") } @@ -153,12 +149,18 @@ func TestComposerRewriteMetadataMinifiedDevReset(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping([]byte(input)) + output, err := h.rewriteMetadata([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } - versions := composerVersions(t, output, "symfony/console") + var result map[string]any + if err := json.Unmarshal(output, &result); err != nil { + t.Fatalf("failed to parse output: %v", err) + } + + packages := result["packages"].(map[string]any) + versions := packages["symfony/console"].([]any) if len(versions) != 2 { t.Fatalf("expected 2 versions, got %d", len(versions)) @@ -166,7 +168,7 @@ func TestComposerRewriteMetadataMinifiedDevReset(t *testing.T) { // Dev version should NOT have inherited "license" or "description" // from the tagged version (the ~dev sentinel resets inheritance). - devVersion := versions[1] + devVersion := versions[1].(map[string]any) if devVersion["version"] != "dev-main" { t.Errorf("dev version = %v, want %q", devVersion["version"], "dev-main") } @@ -214,18 +216,24 @@ func TestComposerRewriteMetadataUnset(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping([]byte(input)) + output, err := h.rewriteMetadata([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } - versions := composerVersions(t, output, "venturecraft/revisionable") + var result map[string]any + if err := json.Unmarshal(output, &result); err != nil { + t.Fatalf("failed to parse output: %v", err) + } + + versions := result["packages"].(map[string]any)["venturecraft/revisionable"].([]any) if len(versions) != 4 { t.Fatalf("expected 4 versions, got %d", len(versions)) } byVersion := map[string]map[string]any{} - for _, vmap := range versions { + for _, v := range versions { + vmap := v.(map[string]any) byVersion[vmap["version"].(string)] = vmap } @@ -262,7 +270,7 @@ func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) { // Minified format where "name" only appears in first version. // When cooldown filters the first version, remaining versions must - // still have the "name" field once Composer expands them. + // still have the "name" field after expansion. input := `{ "minified": "composer/2.0", "packages": { @@ -288,12 +296,18 @@ func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping([]byte(input)) + output, err := h.rewriteMetadata([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } - versions := composerVersions(t, output, "symfony/console") + var result map[string]any + if err := json.Unmarshal(output, &result); err != nil { + t.Fatalf("failed to parse output: %v", err) + } + + packages := result["packages"].(map[string]any) + versions := packages["symfony/console"].([]any) // v7.0.0 should be filtered by cooldown, leaving v6.0.0 and v5.0.0 if len(versions) != 2 { @@ -301,7 +315,8 @@ func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) { } // Both remaining versions must have the "name" field - for _, vmap := range versions { + for _, v := range versions { + vmap := v.(map[string]any) if vmap["name"] != "symfony/console" { t.Errorf("version %v missing name field, got %v", vmap["version"], vmap["name"]) } @@ -317,12 +332,21 @@ func TestComposerRewriteDistURLGitHubZipball(t *testing.T) { proxyURL: "http://localhost:8080", } - url, ok := h.proxyDistURL("symfony/asset", "v7.4.8", - "https://api.github.com/repos/symfony/asset/zipball/d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", "zip") - if !ok { - t.Fatal("dist URL was not rewritten") + vmap := map[string]any{ + "version": "v7.4.8", + "dist": map[string]any{ + "url": "https://api.github.com/repos/symfony/asset/zipball/d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", + "type": "zip", + "shasum": "", + "reference": "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", + }, } + h.rewriteDistURL(vmap, "symfony/asset", "v7.4.8") + + dist := vmap["dist"].(map[string]any) + url := dist["url"].(string) + // The rewritten URL's filename must have a .zip extension if !strings.HasSuffix(url, ".zip") { t.Errorf("rewritten dist URL filename has no .zip extension: %s", url) @@ -352,7 +376,7 @@ func TestComposerRewriteMetadataGitHubZipballFilenames(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping([]byte(input)) + output, err := h.rewriteMetadata([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -375,16 +399,16 @@ func TestComposerRewriteMetadataGitHubZipballFilenames(t *testing.T) { func TestComposerExpandMinifiedSharedDistReferences(t *testing.T) { // When a minified version inherits the dist field from a previous version - // (i.e. it doesn't include its own dist), rewriting must still give each - // version a dist URL naming its own version. + // (i.e. it doesn't include its own dist), expanding + rewriting must not + // corrupt the dist URLs via shared map references. h := &ComposerHandler{ proxy: testProxy(), proxyURL: "http://localhost:8080", } // In this minified payload, v5.3.0 does NOT include a dist field, - // so it inherits v5.4.0's dist. After URL rewriting, each version must - // have its own correct dist URL. + // so it inherits v5.4.0's dist. After expansion and URL rewriting, + // each version must have its own correct dist URL. input := `{ "minified": "composer/2.0", "packages": { @@ -405,18 +429,30 @@ func TestComposerExpandMinifiedSharedDistReferences(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping([]byte(input)) + output, err := h.rewriteMetadata([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } - versions := composerVersions(t, output, "vendor/pkg") + var result map[string]any + if err := json.Unmarshal(output, &result); err != nil { + t.Fatalf("failed to parse output: %v", err) + } + + packages := result["packages"].(map[string]any) + versions := packages["vendor/pkg"].([]any) if len(versions) != 2 { t.Fatalf("expected 2 versions, got %d", len(versions)) } - url1 := distURL(t, versions[0]) - url2 := distURL(t, versions[1]) + v1 := versions[0].(map[string]any) + v2 := versions[1].(map[string]any) + + dist1 := v1["dist"].(map[string]any) + dist2 := v2["dist"].(map[string]any) + + url1 := dist1["url"].(string) + url2 := dist2["url"].(string) // Each version must have its own URL with its own version in the path if !strings.Contains(url1, "/5.4.0/") { @@ -494,29 +530,28 @@ func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) { // OLD behavior: fetching only the regular file fails to resolve the dev // version, which is what produced the 404 before the fix. stableURL := srv.URL + "/p2/phpmd/phpmd.json" - found, err := h.findDownloadURLFromMetadata(ctx, stableURL, pkg, version) + got, err := h.findDownloadURLFromMetadata(ctx, stableURL, pkg, version) if err != nil { t.Fatalf("unexpected error fetching regular metadata: %v", err) } - if found.url != "" { + if got != "" { t.Fatalf("regular metadata unexpectedly contained dev version %q (got %q); "+ - "the test no longer reproduces the original bug", version, found.url) + "the test no longer reproduces the original bug", version, got) } // FIXED behavior: the handler consults the ~dev file (it is first in the // candidate list for dev versions) and resolves the dist URL. - files := h.metadataFilesForVersion(vendor, name, version) - if len(files) == 0 || !strings.HasSuffix(files[0].url, "/p2/phpmd/phpmd~dev.json") || files[0].cacheKey != "phpmd/phpmd~dev" { - t.Fatalf("dev version should consult the ~dev metadata file first, got %v", files) + urls := h.metadataURLsForVersion(vendor, name, version) + if len(urls) == 0 || !strings.HasSuffix(urls[0], "/p2/phpmd/phpmd~dev.json") { + t.Fatalf("dev version should consult the ~dev metadata file first, got %v", urls) } var resolved string - for _, f := range files { - found, err := h.findDownloadURLFromMetadata(ctx, f.url, pkg, version) + for _, u := range urls { + resolved, err = h.findDownloadURLFromMetadata(ctx, u, pkg, version) if err != nil { - t.Fatalf("unexpected error fetching metadata %q: %v", f.url, err) + t.Fatalf("unexpected error fetching metadata %q: %v", u, err) } - resolved = found.url if resolved != "" { break } @@ -527,408 +562,6 @@ func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) { } } -const ( - composerCommitA = "1111111111111111111111111111111111111111" - composerCommitB = "2222222222222222222222222222222222222222" -) - -// setupComposerBranch returns a caching Composer handler whose upstream lists -// distPath as the dist of v/p's dev-master and 1.0.0 and serves archives by -// request URI. -func setupComposerBranch(t *testing.T, distPath string, archives map[string]string) (http.Handler, *Proxy) { - t.Helper() - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/p2/v/p.json" || r.URL.Path == "/p2/v/p~dev.json" { - dist := fmt.Sprintf(`"dist":{"url":%q,"type":"zip"}`, srv.URL+distPath) - _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[{"version":"dev-master",%s},{"version":"1.0.0",%s}]}}`, dist, dist) - return - } - body, ok := archives[r.URL.RequestURI()] - if !ok { - http.NotFound(w, r) - return - } - _, _ = io.WriteString(w, body) - })) - t.Cleanup(srv.Close) - - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - return NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes(), proxy -} - -// cachedComposerFile returns the cached bytes of v/p dev-master's filename, -// or "" when nothing is cached under it. -func cachedComposerFile(t *testing.T, proxy *Proxy, filename string) string { - t.Helper() - cached, err := proxy.GetCachedArtifact(context.Background(), "composer", "v/p", "dev-master", filename) - if err != nil { - t.Fatalf("reading cache for %s: %v", filename, err) - } - if cached == nil { - return "" - } - defer func() { _ = cached.Reader.Close() }() - data, err := io.ReadAll(cached.Reader) - if err != nil { - t.Fatalf("reading cached %s: %v", filename, err) - } - return string(data) -} - -// TestComposerDownloadPinnedBranchCommit checks that a lock file pinning an -// older commit of a branch gets that commit rather than the head the branch's -// metadata lists now, and that the head is not cached under the older name. -// The second round is served from the cache. -func TestComposerDownloadPinnedBranchCommit(t *testing.T) { - tests := []struct { - name, distPath, filename, cached string - }{ - {"github zipball", "/repos/v/p/zipball/%s", "%s.zip", "%s.zip"}, - {"github zipball without .zip", "/repos/v/p/zipball/%s", "%s", "%s"}, - {"bitbucket archive", "/v/p/get/%s.zip", "%s.zip", "%s.zip"}, - {"gitlab archive", "/api/v4/projects/v%%2Fp/repository/archive.zip?sha=%s", "archive.zip?sha=%s", "archive-%s.zip"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - h, proxy := setupComposerBranch(t, fmt.Sprintf(tt.distPath, composerCommitB), map[string]string{ - fmt.Sprintf(tt.distPath, composerCommitA): "commit A", - fmt.Sprintf(tt.distPath, composerCommitB): "commit B", - }) - for range 2 { - for _, c := range []struct{ commit, want string }{ - {composerCommitA, "commit A"}, - {composerCommitB, "commit B"}, - } { - filename, want := fmt.Sprintf(tt.filename, c.commit), c.want - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+filename, nil)) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Errorf("%s: status %d, body %q, want 200 %q", filename, w.Code, w.Body.String(), want) - } - cached := fmt.Sprintf(tt.cached, c.commit) - if got := cachedComposerFile(t, proxy, cached); got != want { - t.Errorf("%s cached as %q, want %q", cached, got, want) - } - } - } - }) - } -} - -// composerMovingBranch is an upstream whose dev-master head can move. It -// serves the archives of commits A and B under distPath and counts how often -// they are fetched. -type composerMovingBranch struct { - h http.Handler - proxy *Proxy - url string - head atomic.Value - fetches atomic.Int32 -} - -func setupComposerMovingBranch(t *testing.T, distPath string) *composerMovingBranch { - t.Helper() - b := &composerMovingBranch{} - b.head.Store(composerCommitA) - archives := map[string]string{ - fmt.Sprintf(distPath, composerCommitA): "commit A", - fmt.Sprintf(distPath, composerCommitB): "commit B", - } - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/p2/v/p.json" || r.URL.Path == "/p2/v/p~dev.json" { - dist := b.url + fmt.Sprintf(distPath, b.head.Load()) - _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[{"version":"dev-master","dist":{"url":%q,"type":"zip"}}]}}`, dist) - return - } - body, ok := archives[r.URL.RequestURI()] - if !ok { - http.NotFound(w, r) - return - } - b.fetches.Add(1) - _, _ = io.WriteString(w, body) - })) - t.Cleanup(srv.Close) - b.url = srv.URL - - b.proxy, _, _, _ = setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) - b.proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - b.h = NewComposerHandlerWithUpstreams(b.proxy, "http://proxy", srv.URL, srv.URL).Routes() - return b -} - -// TestComposerDownloadBranchMovesOn checks a lock file pinning A while the -// branch moves from A to B: A is cached while it is the head and still served -// from the cache afterwards, and B, the head a new lock file pins, comes from -// upstream. -func TestComposerDownloadBranchMovesOn(t *testing.T) { - tests := []struct { - name, distPath, filename, cached string - }{ - {"github zipball", "/repos/v/p/zipball/%s", "%s.zip", "%s.zip"}, - {"gitlab archive", "/api/v4/projects/v%%2Fp/repository/archive.zip?sha=%s", "archive.zip?sha=%s", "archive-%s.zip"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := setupComposerMovingBranch(t, tt.distPath) - get := func(commit, want string, wantFetches int32) { - t.Helper() - filename := fmt.Sprintf(tt.filename, commit) - w := httptest.NewRecorder() - b.h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+filename, nil)) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Errorf("%s: status %d, body %q, want 200 %q", filename, w.Code, w.Body.String(), want) - } - if got := cachedComposerFile(t, b.proxy, fmt.Sprintf(tt.cached, commit)); got != want { - t.Errorf("%s cached as %q, want %q", filename, got, want) - } - if n := b.fetches.Load(); n != wantFetches { - t.Errorf("after %s: %d upstream fetches, want %d", filename, n, wantFetches) - } - } - get(composerCommitA, "commit A", 1) - b.head.Store(composerCommitB) - get(composerCommitA, "commit A", 1) - get(composerCommitB, "commit B", 2) - get(composerCommitB, "commit B", 2) - }) - } -} - -// TestComposerDownloadGitLabSharedName checks that an archive an older proxy -// cached under GitLab's shared archive.zip name is dropped once a commit is -// fetched under its own name. -func TestComposerDownloadGitLabSharedName(t *testing.T) { - distPath := "/api/v4/projects/v%%2Fp/repository/archive.zip?sha=%s" - b := setupComposerMovingBranch(t, distPath) - shared, err := b.proxy.GetOrFetchArtifactFromURL(context.Background(), "composer", "v/p", "dev-master", - "archive.zip", b.url+fmt.Sprintf(distPath, composerCommitB)) - if err != nil { - t.Fatalf("seeding shared entry: %v", err) - } - _ = shared.Reader.Close() - - w := httptest.NewRecorder() - b.h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/archive.zip?sha="+composerCommitA, nil)) - if w.Code != http.StatusOK || w.Body.String() != "commit A" { - t.Errorf("status %d, body %q, want 200 %q", w.Code, w.Body.String(), "commit A") - } - if got := cachedComposerFile(t, b.proxy, "archive.zip"); got != "" { - t.Errorf("archive.zip still cached as %q", got) - } -} - -// TestComposerDownloadCachedFromOtherCommit checks that an archive an older -// proxy cached under a commit's name but fetched from another commit's URL is -// fetched again, while a correct entry is served from the cache. -func TestComposerDownloadCachedFromOtherCommit(t *testing.T) { - zipball := "/repos/v/p/zipball/" - var fetches atomic.Int32 - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/p2/v/p.json", "/p2/v/p~dev.json": - _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[{"version":"dev-master","dist":{"url":%q,"type":"zip"}}]}}`, srv.URL+zipball+composerCommitB) - case zipball + composerCommitA: - fetches.Add(1) - _, _ = io.WriteString(w, "commit A") - case zipball + composerCommitB: - fetches.Add(1) - _, _ = io.WriteString(w, "commit B") - default: - http.NotFound(w, r) - } - })) - t.Cleanup(srv.Close) - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes() - - // What proxy versions before the pinned-commit fix stored: the head - // under the name of the commit the lock file pins. - stale, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "composer", "v/p", "dev-master", - composerCommitA+".zip", srv.URL+zipball+composerCommitB) - if err != nil { - t.Fatalf("seeding stale entry: %v", err) - } - _ = stale.Reader.Close() - fetches.Store(0) - - for range 2 { - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+composerCommitA+".zip", nil)) - if w.Code != http.StatusOK || w.Body.String() != "commit A" { - t.Errorf("status %d, body %q, want 200 %q", w.Code, w.Body.String(), "commit A") - } - } - if got := cachedComposerFile(t, proxy, composerCommitA+".zip"); got != "commit A" { - t.Errorf("cached as %q, want %q", got, "commit A") - } - if n := fetches.Load(); n != 1 { - t.Errorf("upstream archive fetched %d times, want 1", n) - } -} - -// TestComposerDownloadGitLabTaggedRelease checks that a tagged release on -// GitLab is only served as the commit its metadata lists. -func TestComposerDownloadGitLabTaggedRelease(t *testing.T) { - archive := "/api/v4/projects/v%2Fp/repository/archive.zip?sha=" - h, _ := setupComposerBranch(t, archive+composerCommitB, map[string]string{ - archive + composerCommitA: "commit A", - archive + composerCommitB: "commit B", - }) - - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/1.0.0/archive.zip?sha="+composerCommitA, nil)) - if w.Code != http.StatusNotFound { - t.Errorf("other commit: status %d, body %q, want 404", w.Code, w.Body.String()) - } - - w = httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/1.0.0/archive.zip?sha="+composerCommitB, nil)) - if w.Code != http.StatusOK || w.Body.String() != "commit B" { - t.Errorf("listed commit: status %d, body %q, want 200 %q", w.Code, w.Body.String(), "commit B") - } -} - -// TestComposerDownloadBranchWithSlash checks that a branch whose name holds a -// slash gets a dist URL the download route matches. -func TestComposerDownloadBranchWithSlash(t *testing.T) { - const version = "dev-feature/x" - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/zip/x.zip" { - _, _ = io.WriteString(w, "feature") - return - } - _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[{"version":%q,"dist":{"url":%q,"type":"zip"}}]}}`, version, srv.URL+"/zip/x.zip") - })) - t.Cleanup(srv.Close) - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL) - - distURL, ok := h.proxyDistURL("v/p", version, srv.URL+"/zip/x.zip", "zip") - if want := "http://proxy/composer/files/v/p/dev-feature%2Fx/x.zip"; !ok || distURL != want { - t.Fatalf("dist URL = %q, want %q", distURL, want) - } - - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, strings.TrimPrefix(distURL, "http://proxy/composer"), nil)) - if w.Code != http.StatusOK || w.Body.String() != "feature" { - t.Errorf("status %d, body %q, want 200 %q", w.Code, w.Body.String(), "feature") - } -} - -// TestComposerDownloadUnlistedFile checks that a file the metadata no longer -// lists, and that names no commit to fetch instead, is a 404 rather than the -// listed archive cached under the requested name. -func TestComposerDownloadUnlistedFile(t *testing.T) { - tests := []struct { - name, distPath, listed, stale string - }{ - {"plain name", "/dist/main.zip", "main.zip", "old.zip"}, - {"escaped name", "/dist/p%2Bdev-main.zip", "p%2Bdev-main.zip", "old.zip"}, - {"gitlab archive with other query", "/api/v4/projects/v%2Fp/repository/archive.zip?sha=" + composerCommitB, - "archive.zip?sha=" + composerCommitB, "archive.zip?sha=" + composerCommitA + "&path=src"}, - {"branch name for a commit", "/repos/v/p/zipball/" + composerCommitB, composerCommitB + ".zip", "master.zip"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - h, proxy := setupComposerBranch(t, tt.distPath, map[string]string{tt.distPath: "head"}) - - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+tt.stale, nil)) - if w.Code != http.StatusNotFound { - t.Errorf("%s: status %d, body %q, want 404", tt.stale, w.Code, w.Body.String()) - } - staleName, _, _ := strings.Cut(tt.stale, "?") - if got := cachedComposerFile(t, proxy, staleName); got != "" { - t.Errorf("%s cached as %q, want nothing", staleName, got) - } - - w = httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+tt.listed, nil)) - if w.Code != http.StatusOK || w.Body.String() != "head" { - t.Errorf("%s: status %d, body %q, want 200 %q", tt.listed, w.Code, w.Body.String(), "head") - } - }) - } -} - -// TestComposerDownloadTaggedReleaseCommit checks that a tagged release is only -// served as the commit its metadata lists, so a client cannot have another -// commit stored as that release. -func TestComposerDownloadTaggedReleaseCommit(t *testing.T) { - zipball := "/repos/v/p/zipball/" - h, proxy := setupComposerBranch(t, zipball+composerCommitB, map[string]string{ - zipball + composerCommitA: "commit A", - zipball + composerCommitB: "commit B", - }) - - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/1.0.0/"+composerCommitA+".zip", nil)) - if w.Code != http.StatusNotFound { - t.Errorf("other commit: status %d, body %q, want 404", w.Code, w.Body.String()) - } - cached, err := proxy.GetCachedArtifact(context.Background(), "composer", "v/p", "1.0.0", composerCommitA+".zip") - if err != nil || cached != nil { - t.Errorf("other commit cached as 1.0.0: %v, %v", cached, err) - } - - w = httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/1.0.0/"+composerCommitB+".zip", nil)) - if w.Code != http.StatusOK || w.Body.String() != "commit B" { - t.Errorf("listed commit: status %d, body %q, want 200 %q", w.Code, w.Body.String(), "commit B") - } -} - -func TestComposerUpstreamDistURL(t *testing.T) { - const zipball = "https://api.github.com/repos/v/p/zipball/" - const gitlab = "https://gitlab.com/api/v4/projects/v%2Fp/repository/archive.zip?sha=" - sha256A := strings.Repeat("a", 64) - tests := []struct { - name, listed, distType, filename, want string - }{ - {"listed file", zipball + composerCommitB, "zip", composerCommitB + ".zip", zipball + composerCommitB}, - {"older commit", zipball + composerCommitB, "zip", composerCommitA + ".zip", zipball + composerCommitA}, - {"sha256 commit", zipball + composerCommitB, "zip", sha256A + ".zip", zipball + sha256A}, - {"bare listed commit", zipball + composerCommitB, "zip", composerCommitB, zipball + composerCommitB}, - {"bare older commit", zipball + composerCommitB, "zip", composerCommitA, zipball + composerCommitA}, - {"tar dist keeps bare name", zipball + composerCommitB, "tar", composerCommitA, zipball + composerCommitA}, - {"short hash", zipball + composerCommitB, "zip", "1111111.zip", ""}, - {"upper-case hash", zipball + composerCommitB, "zip", strings.ToUpper(sha256A) + ".zip", ""}, - {"other name", "https://example.com/dist/main.zip", "zip", "old.zip", ""}, - {"gitlab older commit", gitlab + composerCommitB, "zip", "archive.zip?sha=" + composerCommitA, gitlab + composerCommitA}, - {"gitlab short hash", gitlab + composerCommitB, "zip", "archive.zip?sha=1111111", ""}, - {"gitlab other archive", gitlab + composerCommitB, "zip", "other.zip?sha=" + composerCommitA, ""}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, ok := upstreamDistURL(tt.listed, tt.distType, tt.filename, true) - if got != tt.want || ok != (tt.want != "") { - t.Errorf("upstreamDistURL(%q, %q) = %q, %v; want %q", tt.listed, tt.filename, got, ok, tt.want) - } - }) - } - if got, ok := upstreamDistURL(zipball+composerCommitB, "zip", composerCommitA+".zip", false); ok { - t.Errorf("tagged release rebuilt to %q, want only the listed archive", got) - } - if got, ok := upstreamDistURL(gitlab+composerCommitB, "zip", "archive.zip?sha="+composerCommitA, false); ok { - t.Errorf("tagged GitLab release rebuilt to %q, want only the listed archive", got) - } -} - func TestComposerRewriteMetadataCooldown(t *testing.T) { now := time.Now() old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339) @@ -959,7 +592,7 @@ func TestComposerRewriteMetadataCooldown(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping([]byte(input)) + output, err := h.rewriteMetadata([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -981,97 +614,3 @@ func TestComposerRewriteMetadataCooldown(t *testing.T) { t.Errorf("expected version 5.0.0, got %v", v["version"]) } } - -func TestComposerRetentionKey(t *testing.T) { - tests := []struct { - key string - want string - ok bool - }{ - {"pkg:composer/symfony/console", "pkg:composer/symfony/console", true}, - {"pkg:composer/Symfony/Console", "pkg:composer/symfony/console", true}, - {"pkg:composer/console", "", false}, - {"pkg:composer/symfony/console/extra", "", false}, - {"pkg:npm/symfony/console", "", false}, - } - for _, tt := range tests { - p, err := purl.Parse(tt.key) - if err != nil { - t.Fatalf("Parse(%s): %v", tt.key, err) - } - got, ok := composerRetentionKey(p) - if ok != tt.ok || (ok && got != tt.want) { - t.Errorf("composerRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) - } - } - if _, ok := composerRetentionKey(nil); ok { - t.Error("composerRetentionKey(nil) accepted") - } -} - -// TestComposerDownloadCooldown checks that an archive still in cooldown is -// withheld when a lock file requests it directly, without the metadata that -// would have filtered it out. In minified metadata a version inherits the -// time of the entry before it. -func TestComposerDownloadCooldown(t *testing.T) { - now := time.Now() - old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339) - recent := now.Add(-1 * time.Hour).Format(time.RFC3339) - - var archiveRequests atomic.Int32 - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - dist := func(version string) string { - return fmt.Sprintf(`"dist":{"url":%q,"type":"zip"}`, srv.URL+"/archives/"+version+".zip") - } - switch r.URL.Path { - case "/p2/v/p.json": - _, _ = fmt.Fprintf(w, `{"minified":"composer/2.0","packages":{"v/p":[`+ - `{"version":"2.0.0","time":%q,%s},`+ - `{"version":"1.9.0",%s},`+ - `{"version":"1.0.0","time":%q,%s}]}}`, - recent, dist("2.0.0"), dist("1.9.0"), old, dist("1.0.0")) - case "/p2/v/p~dev.json": - http.NotFound(w, r) - default: - archiveRequests.Add(1) - _, _ = io.WriteString(w, "archive "+r.URL.Path) - } - })) - t.Cleanup(srv.Close) - - proxy, _, _, _ := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes() - - tests := []struct { - version string - wantStatus int - }{ - {"2.0.0", http.StatusNotFound}, - {"1.9.0", http.StatusNotFound}, - {"1.0.0", http.StatusOK}, - } - for _, tt := range tests { - t.Run(tt.version, func(t *testing.T) { - archiveRequests.Store(0) - req := httptest.NewRequest(http.MethodGet, "/files/v/p/"+tt.version+"/"+tt.version+".zip", nil) - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - - if w.Code != tt.wantStatus { - t.Fatalf("status = %d, want %d (body %q)", w.Code, tt.wantStatus, w.Body.String()) - } - wantRequests := int32(0) - if tt.wantStatus == http.StatusOK { - wantRequests = 1 - } - if got := archiveRequests.Load(); got != wantRequests { - t.Errorf("upstream archive requests = %d, want %d", got, wantRequests) - } - }) - } -} diff --git a/internal/handler/conan.go b/internal/handler/conan.go index dddd79f..53f6428 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -1,25 +1,16 @@ package handler import ( + "fmt" + "io" "net/http" "strings" - - "github.com/git-pkgs/proxy/internal/retention" ) const ( conanUpstream = "https://center.conan.io" ) -// Recipe and package archives are stored under pkg:conan/{name}, with the -// version as the version and the revisions and any user/channel in the -// filename, and purl keeps conan names as they are. A package rule therefore -// covers every version, revision and user/channel of a recipe, and uses the -// default key mapping. -func init() { - retention.Default.Register(retention.Spec{Key: "conan", CanonicalPackage: retention.DefaultCanonical("conan")}) -} - // ConanHandler handles Conan registry protocol requests. type ConanHandler struct { proxy *Proxy @@ -36,13 +27,6 @@ func NewConanHandler(proxy *Proxy, proxyURL string) *ConanHandler { } } -// NewConanHandlerWithUpstream creates a Conan handler with a custom upstream. -func NewConanHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *ConanHandler { - h := NewConanHandler(proxy, proxyURL) - h.upstreamURL = configuredUpstreamURL(upstreamURL, conanUpstream) - return h -} - // Routes returns the HTTP handler for Conan requests. func (h *ConanHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -51,12 +35,13 @@ func (h *ConanHandler) Routes() http.Handler { mux.HandleFunc("GET /v1/ping", h.handlePing) mux.HandleFunc("GET /v2/ping", h.handlePing) - // Recipe and package file downloads, as built by the Conan 2 client - // (ClientV2Router.recipe_file and package_file). The v1 protocol hands out - // signed absolute URLs on the upstream host, so its downloads never pass - // through here. - mux.HandleFunc("GET /v2/conans/{name}/{version}/{user}/{channel}/revisions/{revision}/files/{filename}", h.handleRecipeFile) - mux.HandleFunc("GET /v2/conans/{name}/{version}/{user}/{channel}/revisions/{revision}/packages/{pkgref}/revisions/{pkgrev}/files/{filename}", h.handlePackageFile) + // Recipe file downloads (cache these) + mux.HandleFunc("GET /v1/files/{name}/{version}/{user}/{channel}/{revision}/recipe/{filename}", h.handleRecipeFile) + mux.HandleFunc("GET /v2/files/{name}/{version}/{user}/{channel}/{revision}/recipe/{filename}", h.handleRecipeFile) + + // Package file downloads (cache these) + mux.HandleFunc("GET /v1/files/{name}/{version}/{user}/{channel}/{revision}/package/{pkgref}/{pkgrev}/{filename}", h.handlePackageFile) + mux.HandleFunc("GET /v2/files/{name}/{version}/{user}/{channel}/{revision}/package/{pkgref}/{pkgrev}/{filename}", h.handlePackageFile) // Proxy all other endpoints (metadata, search, etc.) with caching mux.HandleFunc("GET /", h.proxyCached) @@ -79,25 +64,32 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request) revision := r.PathValue("revision") filename := r.PathValue("filename") - // Only the archives are artifacts; the rest goes through the metadata cache. + // Only cache specific files if !h.shouldCacheFile(filename) { - h.proxyCached(w, r) + h.proxyUpstream(w, r) return } + // Conan package name format: name/version@user/channel + packageName := fmt.Sprintf("%s/%s@%s/%s", name, version, user, channel) + h.proxy.Logger.Info("conan recipe download", "name", name, "version", version, "user", user, "channel", channel, "filename", filename) upstreamURL := h.upstreamURL + r.URL.Path - storageFilename := conanStorageFilename(user, channel, revision+"_"+filename) - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL) + // Use revision as part of version for storage + storageVersion := fmt.Sprintf("%s_%s", version, revision) + storageFilename := fmt.Sprintf("recipe_%s", filename) + + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch file") + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch file", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) + ServeArtifact(w, result) } // handlePackageFile serves a package file, fetching and caching from upstream if needed. @@ -111,45 +103,40 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) pkgrev := r.PathValue("pkgrev") filename := r.PathValue("filename") - // Only the archives are artifacts; the rest goes through the metadata cache. + // Only cache specific files if !h.shouldCacheFile(filename) { - h.proxyCached(w, r) + h.proxyUpstream(w, r) return } + packageName := fmt.Sprintf("%s/%s@%s/%s", name, version, user, channel) + h.proxy.Logger.Info("conan package download", "name", name, "version", version, "pkgref", pkgref, "filename", filename) upstreamURL := h.upstreamURL + r.URL.Path - storageFilename := conanStorageFilename(user, channel, revision+"_"+pkgref+"_"+pkgrev+"_"+filename) - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL) + // Use revision and package ref as part of version for storage + storageVersion := fmt.Sprintf("%s_%s_%s_%s", version, revision, pkgref, pkgrev) + storageFilename := fmt.Sprintf("package_%s", filename) + + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch file") + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch file", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) -} - -// conanStorageFilename names a cached archive. Archives are stored under -// pkg:conan/{name}@{version}, so the file name carries the revisions and, when -// set, user and channel, to keep every recipe and binary apart. -func conanStorageFilename(user, channel, file string) string { - if user == "_" && channel == "_" { - return file - } - return user + "@" + channel + "_" + file + ServeArtifact(w, result) } // shouldCacheFile returns true if the file should be cached. func (h *ConanHandler) shouldCacheFile(filename string) bool { - // Cache the large archive files. Since Conan 2.25 they can also be xz or - // zstd (core.upload:compression_format). + // Cache the large archive files cacheFiles := []string{ - "conan_sources.tgz", "conan_sources.txz", "conan_sources.tzst", - "conan_export.tgz", "conan_export.txz", "conan_export.tzst", - "conan_package.tgz", "conan_package.txz", "conan_package.tzst", + "conan_sources.tgz", + "conan_export.tgz", + "conan_package.tgz", } for _, f := range cacheFiles { @@ -162,10 +149,53 @@ func (h *ConanHandler) shouldCacheFile(filename string) bool { // proxyCached forwards a request with metadata caching. func (h *ConanHandler) proxyCached(w http.ResponseWriter, r *http.Request) { - cacheKey := hashedCacheKey(h.upstreamURL, r.URL.Path, r.URL.RawQuery) + cacheKey := strings.TrimPrefix(r.URL.Path, "/") + cacheKey = strings.ReplaceAll(cacheKey, "/", "_") + if r.URL.RawQuery != "" { + cacheKey += "_" + r.URL.RawQuery + } upstreamURL := h.upstreamURL + r.URL.Path if r.URL.RawQuery != "" { upstreamURL += "?" + r.URL.RawQuery } h.proxy.ProxyCached(w, r, upstreamURL, "conan", cacheKey, "*/*") } + +// proxyUpstream forwards a request to conan center without caching. +func (h *ConanHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { + upstreamURL := h.upstreamURL + r.URL.Path + if r.URL.RawQuery != "" { + upstreamURL += "?" + r.URL.RawQuery + } + + h.proxy.Logger.Debug("proxying to upstream", "url", upstreamURL) + + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) + if err != nil { + http.Error(w, "failed to create request", http.StatusInternalServerError) + return + } + + // Copy authorization header for authenticated requests + if auth := r.Header.Get("Authorization"); auth != "" { + req.Header.Set("Authorization", auth) + } + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + h.proxy.Logger.Error("upstream request failed", "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + + // Copy response headers + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) +} diff --git a/internal/handler/conan_test.go b/internal/handler/conan_test.go index cf8038e..a7bd362 100644 --- a/internal/handler/conan_test.go +++ b/internal/handler/conan_test.go @@ -6,9 +6,7 @@ import ( "net/http" "net/http/httptest" "strings" - "sync/atomic" "testing" - "time" ) const testProxyURL = "http://localhost:8080" @@ -30,13 +28,6 @@ func TestConanShouldCacheFile(t *testing.T) { {"conan_sources.tgz", true}, {"conan_export.tgz", true}, {"conan_package.tgz", true}, - {"conan_sources.txz", true}, - {"conan_export.txz", true}, - {"conan_package.txz", true}, - {"conan_sources.tzst", true}, - {"conan_export.tzst", true}, - {"conan_package.tzst", true}, - {"conan_package.tar.gz", false}, {"conanfile.py", false}, {"conanmanifest.txt", false}, {"conaninfo.txt", false}, @@ -94,10 +85,8 @@ func TestConanPingV2(t *testing.T) { } } -func TestConanMetadata(t *testing.T) { - var requests atomic.Int32 +func TestConanProxyUpstream(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) if r.URL.Path != "/v2/conans/search" { w.WriteHeader(http.StatusNotFound) return @@ -112,26 +101,27 @@ func TestConanMetadata(t *testing.T) { })) defer upstream.Close() - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - h := NewConanHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL) - routes := h.Routes() - for range 2 { - req := httptest.NewRequest(http.MethodGet, "/v2/conans/search?q=zlib", nil) - w := httptest.NewRecorder() - routes.ServeHTTP(w, req) - if w.Code != http.StatusOK || w.Body.String() != `{"results":["zlib/1.2.13"]}` { - t.Fatalf("response = %d %q", w.Code, w.Body.String()) - } + h := &ConanHandler{ + proxy: conanTestProxy(), + upstreamURL: upstream.URL, + proxyURL: "http://proxy.local", } - if got := requests.Load(); got != 1 { - t.Errorf("upstream requests = %d, want 1", got) + + req := httptest.NewRequest(http.MethodGet, "/v2/conans/search?q=zlib", nil) + w := httptest.NewRecorder() + h.proxyUpstream(w, req) + + if w.Code != http.StatusOK { + t.Errorf("status = %d, want %d", w.Code, http.StatusOK) + } + + body := w.Body.String() + if !strings.Contains(body, "zlib/1.2.13") { + t.Errorf("response body does not contain expected result: %s", body) } } -func TestConanMetadataNotFound(t *testing.T) { +func TestConanProxyUpstreamNotFound(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) })) @@ -145,16 +135,16 @@ func TestConanMetadataNotFound(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/nonexistent", nil) w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + h.proxyUpstream(w, req) if w.Code != http.StatusNotFound { t.Errorf("status = %d, want %d", w.Code, http.StatusNotFound) } } -func TestConanMetadataCopiesHeaders(t *testing.T) { +func TestConanProxyUpstreamCopiesHeaders(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("ETag", `"metadata"`) + w.Header().Set("X-Custom-Header", "test-value") w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{}`)) @@ -169,17 +159,17 @@ func TestConanMetadataCopiesHeaders(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/test", nil) w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + h.proxyUpstream(w, req) - if w.Header().Get("ETag") != `"metadata"` { - t.Errorf("ETag = %q, want %q", w.Header().Get("ETag"), `"metadata"`) + if w.Header().Get("X-Custom-Header") != "test-value" { + t.Errorf("X-Custom-Header = %q, want %q", w.Header().Get("X-Custom-Header"), "test-value") } } -func TestConanMetadataDoesNotForwardClientAuth(t *testing.T) { +func TestConanProxyUpstreamForwardsAuthHeader(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { auth := r.Header.Get("Authorization") - if auth != "" { + if auth != "Bearer mytoken" { w.WriteHeader(http.StatusUnauthorized) return } @@ -197,14 +187,14 @@ func TestConanMetadataDoesNotForwardClientAuth(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/test", nil) req.Header.Set("Authorization", "Bearer mytoken") w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + h.proxyUpstream(w, req) if w.Code != http.StatusOK { t.Errorf("status = %d, want %d", w.Code, http.StatusOK) } } -func TestConanMetadataBadUpstream(t *testing.T) { +func TestConanProxyUpstreamBadUpstream(t *testing.T) { h := &ConanHandler{ proxy: conanTestProxy(), upstreamURL: "http://127.0.0.1:1", // unreachable @@ -213,7 +203,7 @@ func TestConanMetadataBadUpstream(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/test", nil) w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + h.proxyUpstream(w, req) if w.Code != http.StatusBadGateway { t.Errorf("status = %d, want %d", w.Code, http.StatusBadGateway) @@ -235,7 +225,7 @@ func TestConanRecipeFileNonCacheable(t *testing.T) { proxyURL: "http://proxy.local", } - req := httptest.NewRequest(http.MethodGet, "/v2/conans/zlib/1.2.13/_/_/revisions/abc123/files/conanfile.py", nil) + req := httptest.NewRequest(http.MethodGet, "/v2/files/zlib/1.2.13/_/_/abc123/recipe/conanfile.py", nil) req.SetPathValue("name", "zlib") req.SetPathValue("version", "1.2.13") req.SetPathValue("user", "_") @@ -270,7 +260,7 @@ func TestConanPackageFileNonCacheable(t *testing.T) { proxyURL: "http://proxy.local", } - req := httptest.NewRequest(http.MethodGet, "/v2/conans/zlib/1.2.13/_/_/revisions/abc123/packages/pkgref1/revisions/pkgrev1/files/conaninfo.txt", nil) + req := httptest.NewRequest(http.MethodGet, "/v2/files/zlib/1.2.13/_/_/abc123/package/pkgref1/pkgrev1/conaninfo.txt", nil) req.SetPathValue("name", "zlib") req.SetPathValue("version", "1.2.13") req.SetPathValue("user", "_") @@ -293,173 +283,6 @@ func TestConanPackageFileNonCacheable(t *testing.T) { } } -// The Conan 2 client downloads recipe and package files over these routes; -// the archives must be stored as artifacts and served from the cache after. -func TestConanV2FileRoutesCacheArchives(t *testing.T) { - tests := []struct { - name string - path string - filename string - }{ - {"recipe", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", - "rrev1_conan_sources.tgz"}, - {"package", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tgz", - "rrev1_pkgid1_prev1_conan_package.tgz"}, - {"recipe xz", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_export.txz", - "rrev1_conan_export.txz"}, - {"package zstd", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tzst", - "rrev1_pkgid1_prev1_conan_package.tzst"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, db, _, fetcher := setupTestProxy(t) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Errorf("archive %s was proxied instead of fetched as an artifact", r.URL.Path) - http.Error(w, "unexpected", http.StatusInternalServerError) - })) - defer upstream.Close() - proxy.HTTPClient = upstream.Client() - - h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - for i := range 2 { - fetcher.artifact = artifactBody("archive bytes") - fetcher.fetchCalled = false - - resp, err := http.Get(srv.URL + tt.path) - if err != nil { - t.Fatalf("request %d failed: %v", i+1, err) - } - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - - if resp.StatusCode != http.StatusOK || string(body) != "archive bytes" { - t.Fatalf("request %d: status = %d, body = %q, want 200 %q", i+1, resp.StatusCode, body, "archive bytes") - } - if fetcher.fetchCalled != (i == 0) { - t.Errorf("request %d: fetched = %v, want %v", i+1, fetcher.fetchCalled, i == 0) - } - if i == 0 && fetcher.fetchedURL != upstream.URL+tt.path { - t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, upstream.URL+tt.path) - } - } - - recordedStoragePath(t, db, "pkg:conan/zlib@1.3.1", tt.filename) - }) - } -} - -// References that differ only in user and channel share pkg:conan/zlib@1.3.1 -// but must not share files, even with the same recipe revision. -func TestConanUserChannelKeptApart(t *testing.T) { - proxy, db, _, fetcher := setupTestProxy(t) - h := NewConanHandlerWithUpstream(proxy, testProxyURL, "http://upstream.invalid") - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - for _, ref := range []struct{ path, body, filename string }{ - {"/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", - "center sources", "rrev1_conan_sources.tgz"}, - {"/v2/conans/zlib/1.3.1/acme/stable/revisions/rrev1/files/conan_sources.tgz", - "acme sources", "acme@stable_rrev1_conan_sources.tgz"}, - } { - fetcher.artifact = artifactBody(ref.body) - fetcher.fetchCalled = false - - resp, err := http.Get(srv.URL + ref.path) - if err != nil { - t.Fatalf("GET %s failed: %v", ref.path, err) - } - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - - if string(body) != ref.body || !fetcher.fetchCalled { - t.Errorf("GET %s: body = %q, fetched = %v, want %q from upstream", ref.path, body, fetcher.fetchCalled, ref.body) - } - recordedStoragePath(t, db, "pkg:conan/zlib@1.3.1", ref.filename) - } -} - -// A denylist entry for a Conan version blocks its recipe and binary archives. -func TestConanDenylistMatchesVersion(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - setTestDenylist(t, proxy, "pkg:conan/zlib@1.3.1") - h := NewConanHandlerWithUpstream(proxy, testProxyURL, "http://upstream.invalid") - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - for _, path := range []string{ - "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", - "/v2/conans/zlib/1.3.1/acme/stable/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tgz", - } { - fetcher.artifact = artifactBody("archive bytes") - resp, err := http.Get(srv.URL + path) - if err != nil { - t.Fatalf("GET %s failed: %v", path, err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusForbidden { - t.Errorf("GET %s: status = %d, want %d", path, resp.StatusCode, http.StatusForbidden) - } - } - if fetcher.fetchCalled { - t.Error("denied archive was fetched from upstream") - } -} - -// The small files next to the archives keep going through the metadata cache. -func TestConanV2FileRoutesKeepSmallFilesAsMetadata(t *testing.T) { - for name, path := range map[string]string{ - "recipe": "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conanmanifest.txt", - "package": "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conaninfo.txt", - } { - t.Run(name, func(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - if r.URL.Path != path { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "text/plain") - _, _ = w.Write([]byte("small file")) - })) - defer upstream.Close() - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - - h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - for i := range 2 { - resp, err := http.Get(srv.URL + path) - if err != nil { - t.Fatalf("request %d failed: %v", i+1, err) - } - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if resp.StatusCode != http.StatusOK || string(body) != "small file" { - t.Fatalf("request %d: status = %d, body = %q, want 200 %q", i+1, resp.StatusCode, body, "small file") - } - } - - if got := requests.Load(); got != 1 { - t.Errorf("upstream requests = %d, want 1 (second served from the metadata cache)", got) - } - if fetcher.fetchCalled { - t.Error("small file was fetched as an artifact") - } - }) - } -} - func TestConanRoutes(t *testing.T) { h := &ConanHandler{ proxy: conanTestProxy(), @@ -488,9 +311,9 @@ func TestConanRoutes(t *testing.T) { } } -func TestConanMetadataPreservesQueryString(t *testing.T) { +func TestConanProxyUpstreamPreservesQueryString(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("q") != "boost" || r.URL.Query().Get("page") != "2" { + if r.URL.Query().Get("q") != "boost" && r.URL.Query().Get("page") != "2" { w.WriteHeader(http.StatusBadRequest) return } @@ -507,14 +330,14 @@ func TestConanMetadataPreservesQueryString(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/search?q=boost&page=2", nil) w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + h.proxyUpstream(w, req) if w.Code != http.StatusOK { t.Errorf("status = %d, want %d", w.Code, http.StatusOK) } } -func TestConanMetadataLargeResponse(t *testing.T) { +func TestConanProxyUpstreamLargeResponse(t *testing.T) { largeBody := strings.Repeat("x", 1024*1024) upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -530,7 +353,7 @@ func TestConanMetadataLargeResponse(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/test", nil) w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + h.proxyUpstream(w, req) if w.Code != http.StatusOK { t.Errorf("status = %d, want %d", w.Code, http.StatusOK) @@ -565,7 +388,7 @@ func TestNewConanHandlerNoTrailingSlash(t *testing.T) { } } -func TestConanMetadataNoAuthHeaderWhenNotProvided(t *testing.T) { +func TestConanProxyUpstreamNoAuthHeaderWhenNotProvided(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { auth := r.Header.Get("Authorization") if auth != "" { @@ -585,14 +408,14 @@ func TestConanMetadataNoAuthHeaderWhenNotProvided(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/test", nil) w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + h.proxyUpstream(w, req) if w.Code != http.StatusOK { t.Errorf("status = %d, want %d", w.Code, http.StatusOK) } } -func TestConanMetadataCopiesBody(t *testing.T) { +func TestConanProxyUpstreamCopiesBody(t *testing.T) { expected := `{"name":"zlib","version":"1.2.13","user":"_","channel":"_"}` upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -609,7 +432,7 @@ func TestConanMetadataCopiesBody(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/zlib/1.2.13/_/_/latest", nil) w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + h.proxyUpstream(w, req) if w.Code != http.StatusOK { t.Errorf("status = %d, want %d", w.Code, http.StatusOK) @@ -621,7 +444,7 @@ func TestConanMetadataCopiesBody(t *testing.T) { } } -func TestConanMetadataPreservesStatusCodes(t *testing.T) { +func TestConanProxyUpstreamPreservesStatusCodes(t *testing.T) { codes := []int{ http.StatusOK, http.StatusNotFound, @@ -642,7 +465,7 @@ func TestConanMetadataPreservesStatusCodes(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/test", nil) w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + h.proxyUpstream(w, req) if w.Code != code { t.Errorf("status = %d, want %d", w.Code, code) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index c9db7a7..cfa20c8 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -1,24 +1,12 @@ package handler import ( - "bufio" - "bytes" - "compress/gzip" - "crypto/sha256" - "encoding/hex" "encoding/json" - "errors" - "fmt" - "hash" "io" "net/http" - "slices" "strings" - "sync" "time" - "github.com/git-pkgs/proxy/internal/jsonstream" - "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" ) @@ -27,46 +15,11 @@ const ( minCondaParts = 3 // name-version-build requires at least 3 hyphen-separated parts ) -// Packages are stored under the name {channel}/{name}, which purl writes as -// pkg:conda/{channel}%2F{name}. Package rules name the channel as the -// namespace, pkg:conda/conda-forge/numpy, and are mapped to that form. -func init() { - retention.Default.Register(retention.Spec{Key: "conda", CanonicalPackage: condaRetentionKey}) -} - -// condaRetentionKey maps a package rule to the stored package PURL. It -// takes the channel from the namespace, or from the name in the stored -// form. Downloads are routed as /{channel}/{arch}/{filename}, so the channel -// is always one path segment; a key without a channel, with more segments -// or with an empty one is refused, since no stored package looks like that. -// The package name is lowercased the way conda treats it; the channel stays -// as the URL had it. -func condaRetentionKey(p *purl.PURL) (string, bool) { - if p == nil { - return "", false - } - full := p.Name - if p.Namespace != "" { - full = p.Namespace + "/" + p.Name - } - segments := strings.Split(full, "/") - if len(segments) != 2 || slices.Contains(segments, "") { - return "", false - } - channel, name := segments[0], strings.ToLower(segments[1]) - return retention.DefaultCanonical("conda")(purl.New(p.Type, "", channel+"/"+name, "", nil)) -} - // CondaHandler handles Conda/Anaconda registry protocol requests. type CondaHandler struct { proxy *Proxy upstreamURL string proxyURL string - - // repodataStates holds, per metadata cache key, what is known of the - // last filtered copy of a repodata document. Guarded by repodataMu. - repodataMu sync.Mutex - repodataStates map[string]repodataState } // NewCondaHandler creates a new Conda protocol handler. @@ -78,27 +31,14 @@ func NewCondaHandler(proxy *Proxy, proxyURL string) *CondaHandler { } } -// NewCondaHandlerWithUpstream creates a Conda handler with a custom upstream. -func NewCondaHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *CondaHandler { - h := NewCondaHandler(proxy, proxyURL) - h.upstreamURL = configuredUpstreamURL(upstreamURL, condaUpstream) - return h -} - // Routes returns the HTTP handler for Conda requests. func (h *CondaHandler) Routes() http.Handler { mux := http.NewServeMux() // Channel index (repodata) mux.HandleFunc("GET /{channel}/{arch}/repodata.json", h.handleRepodata) - mux.HandleFunc("GET /{channel}/{arch}/repodata.json.bz2", h.handleRepodataVariant) + mux.HandleFunc("GET /{channel}/{arch}/repodata.json.bz2", h.proxyCached) mux.HandleFunc("GET /{channel}/{arch}/current_repodata.json", h.handleRepodata) - mux.HandleFunc("GET /{channel}/{arch}/repodata_from_packages.json", h.handleRepodata) - - // Sharded repodata (CEP 16) - mux.HandleFunc("GET /{channel}/{arch}/"+condaShardIndexFile, h.handleShardIndex) - mux.HandleFunc("GET /{channel}/{arch}/shards/{filename}", h.handleShard) - mux.HandleFunc("GET /{channel}/{arch}/"+condaCooldownShardDir+"/{bucket}/{rest...}", h.handleCooldownShard) // Package downloads (cache these) mux.HandleFunc("GET /{channel}/{arch}/{filename}", h.handleDownload) @@ -112,17 +52,8 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { arch := r.PathValue("arch") filename := r.PathValue("filename") - if strings.HasSuffix(filename, condaShardSuffix) { - h.handleShard(w, r) - return - } - // Only cache actual package files if !h.isPackageFile(filename) { - if isCondaRepodataVariant(filename) && h.cooldownEnabled() { - refuseUnfilteredRepodata(w) - return - } h.proxyUpstream(w, r) return } @@ -136,31 +67,19 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // Include channel in package name packageName := channel + "/" + name - h.proxy.Logger.Debug("conda download request", + h.proxy.Logger.Info("conda download request", "channel", channel, "arch", arch, "name", name, "version", version) - // Builds and subdirs of one version each have their own timestamp, so - // the check is per file and keeps no time in the shared versions row. - // Subdirs distinguish cached files with the same filename. - cacheFilename := arch + "/" + filename - if h.proxy.withholdInCooldown(w, r, downloadVersion{ - ecosystem: "conda", policyName: name, version: version, cacheName: packageName, filename: cacheFilename, - lookup: func() (time.Time, bool, bool, error) { - published, ok, err := h.publishTime(r, channel, arch, filename) - return published, ok, false, err - }, - }) { - return - } - upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, cacheFilename, upstreamURL) + + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, filename, upstreamURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch package", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) + ServeArtifact(w, result) } // isPackageFile returns true if the filename is a Conda package. @@ -169,15 +88,9 @@ func (h *CondaHandler) isPackageFile(filename string) bool { } // parseFilename extracts name and version from a Conda package filename. -func (h *CondaHandler) parseFilename(filename string) (name, version string) { - return CondaFilenameIdentity(filename) -} - -// CondaFilenameIdentity returns the name and version a download of filename -// is cached under, or empty strings when it is not a package file name. // Conda filenames are: {name}-{version}-{build}.{ext} // e.g., "numpy-1.24.0-py311h64a7726_0.conda" -func CondaFilenameIdentity(filename string) (name, version string) { +func (h *CondaHandler) parseFilename(filename string) (name, version string) { // Remove extension base := filename for _, ext := range []string{".conda", ".tar.bz2"} { @@ -211,90 +124,15 @@ func CondaFilenameIdentity(filename string) (name, version string) { return name, version } -// isCondaRepodataVariant reports whether filename is a form of a subdir's -// repodata other than the plain JSON documents the proxy filters, such as -// repodata.json.zst, repodata.json.bz2 or repodata.jlap patches. Package -// files are told apart by the caller. -func isCondaRepodataVariant(filename string) bool { - return strings.HasPrefix(filename, "repodata") || strings.HasPrefix(filename, "current_repodata") -} - -// handleRepodataVariant serves repodata.json.bz2 from the metadata cache, -// unless cooldown is on: the proxy filters only the JSON, so a compressed -// copy would list versions still in cooldown. -func (h *CondaHandler) handleRepodataVariant(w http.ResponseWriter, r *http.Request) { - if h.cooldownEnabled() { - refuseUnfilteredRepodata(w) - return - } - h.proxyCached(w, r) -} - -// refuseUnfilteredRepodata answers 404 for a repodata variant the proxy -// can't filter. conda and mamba then fall back to repodata.json. -func refuseUnfilteredRepodata(w http.ResponseWriter) { - http.Error(w, "only repodata.json is served while cooldown is enabled", http.StatusNotFound) -} - // handleRepodata proxies repodata.json, applying cooldown filtering when enabled. -// -// With cache_metadata the document is read through the metadata cache, as -// without cooldown, so within metadata_ttl it is filtered from storage -// without asking upstream, and after it revalidated with its ETag. A stored -// copy is streamed through the filter rather than read into memory; a -// document just fetched is held compressed, as the metadata cache holds every -// document it fetches. The decompressed document, hundreds of MB for -// conda-forge, is never held whole. func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) { - if !h.cooldownEnabled() { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { h.proxyCached(w, r) return } upstreamURL := h.upstreamURL + r.URL.Path - p := h.proxy - if !p.CacheMetadata || p.DB == nil || p.Storage == nil { - h.streamRepodata(w, r, upstreamURL) - return - } - cacheKey := h.metadataCacheKey(r.URL.Path) - now := time.Now() - if entry := p.freshMetadataEntry("conda", cacheKey); entry != nil { - if body, err := p.openMetadataEntry(r.Context(), "conda", cacheKey, entry); err == nil { - defer func() { _ = body.Close() }() - p.recordStreamedMetadataHit("conda", entry) - digest := entry.ContentDigest.String - if !h.repodataNotModified(w, r, cacheKey, digest, now) { - h.writeFilteredRepodata(w, upstreamURL, cacheKey, digest, body, entry.ContentEncoding.String, now) - } - return - } - } - - // The same key and encoding as proxyCached, so with and without cooldown - // the proxy keeps one copy of the document. - res := p.lookupMetadata(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "gzip", nil) - if errors.Is(res.err, ErrMetadataTooLarge) { - // Too large to cache even compressed: stream it as without the cache. - p.Logger.Warn("conda repodata exceeds the metadata size limit, streaming it uncached", "url", upstreamURL) - h.streamRepodata(w, r, upstreamURL) - return - } - if res.err != nil { - p.writeMetadataFetchError(w, res.err) - return - } - markStale(w, res.stale) - if h.repodataNotModified(w, r, cacheKey, res.digest, now) { - return - } - h.writeFilteredRepodata(w, upstreamURL, cacheKey, res.digest, bytes.NewReader(res.body), res.contentEncoding, now) -} - -// streamRepodata filters repodata.json as it streams from upstream, without -// the metadata cache. -func (h *CondaHandler) streamRepodata(w http.ResponseWriter, r *http.Request, upstreamURL string) { h.proxy.Logger.Debug("fetching repodata for cooldown filtering", "url", upstreamURL) req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) @@ -306,9 +144,6 @@ func (h *CondaHandler) streamRepodata(w http.ResponseWriter, r *http.Request, up resp, err := h.proxy.HTTPClient.Do(req) if err != nil { - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return @@ -316,342 +151,92 @@ func (h *CondaHandler) streamRepodata(w http.ResponseWriter, r *http.Request, up defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - h.proxy.relayResponse(w, r, resp, nil) + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) return } - // Explicitly requesting gzip disables the transport's automatic decoding, - // so writeFilteredRepodata decodes it. - h.writeFilteredRepodata(w, upstreamURL, "", "", resp.Body, resp.Header.Get(headerContentEncoding), time.Now()) -} - -// writeFilteredRepodata writes repodata.json from src, decoded by -// contentEncoding, without the entries in cooldown at now. digest identifies -// the document in the metadata cache under cacheKey, and is empty when it -// isn't cached; when known, the ETag of the filtered copy is remembered for -// the next request. -func (h *CondaHandler) writeFilteredRepodata(w http.ResponseWriter, upstreamURL, cacheKey, digest string, src io.Reader, contentEncoding string, now time.Time) { - // Cooldown needs JSON, so decode here and bound the decompressed size too. - reader := src - if strings.EqualFold(strings.TrimSpace(contentEncoding), "gzip") { - decoded, err := gzip.NewReader(src) - if err != nil { - http.Error(w, "failed to decode response", http.StatusBadGateway) - return - } - defer func() { _ = decoded.Close() }() - reader = decoded - } - // conda-forge's linux-64 repodata.json is several hundred MB, so it is - // filtered as it streams rather than read through ReadMetadata. Once - // output has started, an error aborts the response so the client never - // takes a truncated or unfiltered index as complete. - out := &countingWriter{w: w} - buffered := bufio.NewWriterSize(out, repodataWriteBuffer) - w.Header().Set(headerContentType, "application/json") - filter := h.newRepodataFilter(now) - err := h.filterRepodataWith(io.LimitReader(reader, maxRepodataSize), buffered, filter) - if err == nil { - err = buffered.Flush() - } + body, err := h.proxy.ReadMetadata(resp.Body) if err != nil { - h.proxy.Logger.Error("failed to filter conda repodata", "url", upstreamURL, "error", err) - if out.n == 0 { - http.Error(w, "failed to read response", http.StatusBadGateway) - return - } - panic(http.ErrAbortHandler) - } - if digest != "" { - h.rememberRepodata(cacheKey, digest, filter) - } -} - -// repodataState is what the proxy remembers of the last filtered copy of a -// cached repodata document: which document it was made from, its ETag, and -// until when filtering that document again gives the same copy. -type repodataState struct { - digest string - etag string - // until is when the first entry left out leaves cooldown, or zero when - // none was left out. - until time.Time -} - -// repodataNotModified sets the ETag of the filtered copy of the document -// identified by digest when the proxy knows it, and answers a conditional -// request it matches with 304, reporting whether it did. The ETag is known -// once the document has been filtered, until an entry it left out leaves -// cooldown. Filtering a large document costs seconds, and sending it -// hundreds of MB, so this is what makes revalidating clients cheap. -func (h *CondaHandler) repodataNotModified(w http.ResponseWriter, r *http.Request, cacheKey, digest string, now time.Time) bool { - if digest == "" { - return false - } - h.repodataMu.Lock() - state, ok := h.repodataStates[cacheKey] - h.repodataMu.Unlock() - if !ok || state.digest != digest || (!state.until.IsZero() && !now.Before(state.until)) { - return false - } - w.Header().Set(headerETag, state.etag) - if ifNoneMatchHits(r.Header.Get("If-None-Match"), state.etag) { - w.WriteHeader(http.StatusNotModified) - return true - } - return false -} - -// rememberRepodata records the ETag of a filtered copy of the document -// identified by digest. The copy depends only on the document, the startup -// configuration and which entries were left out, and those stay left out -// until the first of them leaves cooldown. -func (h *CondaHandler) rememberRepodata(cacheKey, digest string, filter *repodataFilter) { - if filter.unknownUntil { + http.Error(w, "failed to read response", http.StatusInternalServerError) return } - etag := keptVersionsETag(digest, "", cacheKey, []string{hex.EncodeToString(filter.withheld.Sum(nil))}) - h.repodataMu.Lock() - defer h.repodataMu.Unlock() - if h.repodataStates == nil { - h.repodataStates = make(map[string]repodataState) - } - h.repodataStates[cacheKey] = repodataState{digest: digest, etag: etag, until: filter.until} -} -// countingWriter counts the bytes written through it, such as whether -// anything has reached the response yet. -type countingWriter struct { - w io.Writer - n int64 -} - -func (c *countingWriter) Write(p []byte) (int, error) { - n, err := c.w.Write(p) - c.n += int64(n) - return n, err -} - -// condaMaxSecondsTimestamp is the last second of year 9999. Like conda, a -// larger repodata timestamp is read as milliseconds and a smaller one as -// seconds, which some older entries use. -const condaMaxSecondsTimestamp = 253402300799 - -// condaTime returns the time a repodata timestamp names. -func condaTime(timestamp float64) time.Time { - if timestamp > condaMaxSecondsTimestamp { - return time.UnixMilli(int64(timestamp)).Truncate(time.Second) - } - return time.Unix(int64(timestamp), 0) -} - -const ( - repodataWriteBuffer = 64 << 10 - // maxRepodataSize stops a runaway upstream body. It is well above the - // largest conda-forge index, which is under 500 MB. - maxRepodataSize = 4 << 30 -) - -// filterRepodata copies repodata.json from r to w, leaving out entries in -// packages and packages.conda that are still inside their cooldown. Kept -// entries and every other member are written with their upstream bytes. -func (h *CondaHandler) filterRepodata(r io.Reader, w io.Writer) error { - return h.filterRepodataWith(r, w, h.newRepodataFilter(time.Now())) -} - -// filterRepodataWith is filterRepodata deciding by filter. -func (h *CondaHandler) filterRepodataWith(r io.Reader, w io.Writer, filter *repodataFilter) error { - return jsonstream.CopyObject(r, w, func(m *jsonstream.Member) error { - if m.Key != "packages" && m.Key != "packages.conda" { - return m.Copy() - } - return m.Object(func(entry *jsonstream.Member) error { - raw, err := entry.Raw() - if err != nil { - return err - } - if filter.inCooldown(entry.Key, raw) { - return nil - } - return entry.Write(raw) - }) - }) -} - -type repodataEntry struct { - Name string `json:"name"` - Version string `json:"version"` - Timestamp float64 `json:"timestamp"` -} - -// repodataFilter decides which repodata entries cooldown leaves out, all as -// of one time, and records what it left out, which with the document fixes -// the filtered copy. -type repodataFilter struct { - h *CondaHandler - now time.Time - // withheld hashes the filenames left out. - withheld hash.Hash - // until is when the first entry left out leaves cooldown. - until time.Time - // unknownUntil is set when an entry was left out without a time it - // leaves cooldown, so until can't be relied on. - unknownUntil bool -} - -func (h *CondaHandler) newRepodataFilter(now time.Time) *repodataFilter { - return &repodataFilter{h: h, now: now, withheld: sha256.New()} -} - -// inCooldown reports whether a repodata entry was published too recently. -// Entries without a name or timestamp are kept. -func (f *repodataFilter) inCooldown(filename string, raw []byte) bool { - var entry repodataEntry - if err := json.Unmarshal(raw, &entry); err != nil { - return false - } - if entry.Name == "" || entry.Timestamp == 0 { - return false - } - publishedAt := condaTime(entry.Timestamp) - decision := f.h.proxy.cooldown().Evaluate("conda", canonicalPackagePURL("conda", entry.Name), publishedAt, f.now) - if decision.Allowed { - return false - } - f.h.proxy.Logger.Info("cooldown: filtering conda package", - "name", entry.Name, "version", entry.Version, "filename", filename) - _, _ = fmt.Fprintf(f.withheld, "%d:%s", len(filename), filename) - switch { - case decision.AvailableAt.IsZero(): - f.unknownUntil = true - case f.until.IsZero() || decision.AvailableAt.Before(f.until): - f.until = decision.AvailableAt - } - return true -} - -// recordInCooldown reports whether a package record with a repodata -// timestamp was published too recently. Records without a name or -// timestamp are kept. -func (h *CondaHandler) recordInCooldown(filename, name, version string, timestamp float64) bool { - if name == "" || timestamp == 0 { - return false - } - publishedAt := condaTime(timestamp) - if h.proxy.cooldown().IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) { - return false - } - h.proxy.Logger.Info("cooldown: filtering conda package", - "name", name, "version", version, "filename", filename) - return true -} - -// metadataCacheKey is the metadata cache key of a request path. -func (h *CondaHandler) metadataCacheKey(urlPath string) string { - return hashedCacheKey(h.upstreamURL, urlPath) -} - -// publishTime streams compressed repodata when it exceeds metadata_max_size. -func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { - repodataPath := "/" + channel + "/" + arch + "/repodata.json" - cacheKey := h.metadataCacheKey(repodataPath) - index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, func() metadataResult { - return h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) - }, func(res metadataResult) (map[string]time.Time, error) { - if errors.Is(res.err, ErrMetadataTooLarge) { - return h.streamRepodataIndex(r, h.upstreamURL+repodataPath) - } - if res.err != nil { - return nil, res.err - } - return h.indexRepodata(res.body, res.contentEncoding) - }) + filtered, err := h.applyCooldownFiltering(body) if err != nil { - return time.Time{}, false, err + h.proxy.Logger.Warn("failed to filter repodata, proxying original", "error", err) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + return } - published, ok := index[filename] - return published, ok, nil + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(filtered) } -// indexRepodata indexes a cached repodata.json body. -func (h *CondaHandler) indexRepodata(body []byte, encoding string) (map[string]time.Time, error) { - var reader io.Reader = bytes.NewReader(body) - if strings.EqualFold(strings.TrimSpace(encoding), "gzip") { - decoded, err := gzip.NewReader(reader) - if err != nil { - return nil, err - } - reader = decoded - } - return readRepodataIndex(reader) -} +// condaTimestampDivisor converts Conda's millisecond timestamps to seconds. +const condaTimestampDivisor = 1000 -// streamRepodataIndex indexes repodata.json as it streams from upstream. -func (h *CondaHandler) streamRepodataIndex(r *http.Request, upstreamURL string) (map[string]time.Time, error) { - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) - if err != nil { +// applyCooldownFiltering removes entries from repodata.json that were +// published too recently based on their timestamp field. +func (h *CondaHandler) applyCooldownFiltering(body []byte) ([]byte, error) { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return body, nil + } + + var repodata map[string]any + if err := json.Unmarshal(body, &repodata); err != nil { return nil, err } - req.Header.Set(headerAcceptEncoding, "gzip") - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("repodata returned %d", resp.StatusCode) - } - var reader io.Reader = resp.Body - if strings.EqualFold(strings.TrimSpace(resp.Header.Get(headerContentEncoding)), "gzip") { - decoded, err := gzip.NewReader(resp.Body) - if err != nil { - return nil, err - } - defer func() { _ = decoded.Close() }() - reader = decoded - } - return readRepodataIndex(reader) -} -// readRepodataIndex maps each package file in a repodata.json to its -// timestamp, leaving out files without one. It reads the document as it -// streams, under the same bound as filterRepodata, so the decompressed size -// is not limited by metadata_max_size. -func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { - index := make(map[string]time.Time) - err := jsonstream.CopyObject(io.LimitReader(r, maxRepodataSize), io.Discard, func(m *jsonstream.Member) error { - if m.Key != "packages" && m.Key != "packages.conda" { - return m.Copy() + for _, key := range []string{"packages", "packages.conda"} { + packages, ok := repodata[key].(map[string]any) + if !ok { + continue } - return m.Object(func(entry *jsonstream.Member) error { - raw, err := entry.Raw() - if err != nil { - return err + + for filename, entry := range packages { + entryMap, ok := entry.(map[string]any) + if !ok { + continue } - var e repodataEntry - if json.Unmarshal(raw, &e) == nil && e.Timestamp != 0 { - index[entry.Key] = condaTime(e.Timestamp) + + ts, ok := entryMap["timestamp"].(float64) + if !ok || ts == 0 { + continue } - return nil - }) - }) - if err != nil { - return nil, err + + publishedAt := time.Unix(int64(ts)/condaTimestampDivisor, 0) + + name, _ := entryMap["name"].(string) + if name == "" { + continue + } + + packagePURL := purl.MakePURLString("conda", name, "") + + if !h.proxy.Cooldown.IsAllowed("conda", packagePURL, publishedAt) { + version, _ := entryMap["version"].(string) + h.proxy.Logger.Info("cooldown: filtering conda package", + "name", name, "version", version, "filename", filename) + delete(packages, filename) + } + } } - return index, nil + + return json.Marshal(repodata) } // proxyCached forwards a metadata request with caching. func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) { - cacheKey := h.metadataCacheKey(r.URL.Path) - // Large JSON indexes can exceed the metadata limit uncompressed. Conda - // clients decode Content-Encoding, so cache and relay gzip verbatim. The - // already-compressed repodata.json.bz2 route must stay on identity. - acceptEncoding := "identity" - if strings.HasSuffix(r.URL.Path, ".json") { - acceptEncoding = "gzip" - } - h.proxy.proxyCachedWithEncoding(w, r, h.upstreamURL+r.URL.Path, "conda", cacheKey, acceptEncoding, nil, "*/*") + cacheKey := strings.TrimPrefix(r.URL.Path, "/") + cacheKey = strings.ReplaceAll(cacheKey, "/", "_") + h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "conda", cacheKey, "*/*") } // proxyUpstream forwards a request to Anaconda without caching. diff --git a/internal/handler/conda_encoding_test.go b/internal/handler/conda_encoding_test.go deleted file mode 100644 index a8f3a79..0000000 --- a/internal/handler/conda_encoding_test.go +++ /dev/null @@ -1,315 +0,0 @@ -package handler - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -func TestCondaRepodataGzip(t *testing.T) { - // Model a large index without allocating hundreds of megabytes: only its - // compressed representation fits the configured metadata limit. - plain := []byte(`{"packages":{},"padding":"` + strings.Repeat("x", 8192) + `"}`) - compressed := gzipPayload(t, plain) - for _, tt := range []struct { - filename string - mode string - }{ - {"repodata.json", "stream"}, - {"repodata.json", "cached"}, - {"repodata.json", "stale"}, - {"current_repodata.json", "stream"}, - {"current_repodata.json", "cached"}, - {"current_repodata.json", "stale"}, - } { - t.Run(tt.filename+"/"+tt.mode, func(t *testing.T) { - upstream := newGzipWhenAskedUpstream(plain, compressed) - defer upstream.Close() - proxy, db, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.CacheMetadata = tt.mode != "stream" - proxy.MetadataMaxSize = int64(len(compressed)) - if tt.mode == "cached" { - proxy.MetadataTTL = time.Hour - } - h := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL) - handler := h.Routes() - serve := func() *httptest.ResponseRecorder { - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+tt.filename, nil)) - return w - } - assertGzipResponse(t, "first", serve(), compressed) - if got := upstream.sawAcceptEncoding(); got != "gzip" { - t.Fatalf("upstream Accept-Encoding = %q, want gzip", got) - } - if tt.mode == "stream" { - return - } - entry, err := db.GetMetadataCache("conda", h.metadataCacheKey("/conda-forge/linux-64/"+tt.filename)) - if err != nil { - t.Fatal(err) - } - if entry == nil || entry.ContentEncoding.String != "gzip" || entry.Size.Int64 != int64(len(compressed)) { - t.Fatalf("cache entry does not describe compressed bytes: %+v", entry) - } - upstream.available.Store(false) - assertGzipResponse(t, "offline replay", serve(), compressed) - wantRequests := int32(1) - if tt.mode == "stale" { - wantRequests = 2 - } - if got := upstream.requests.Load(); got != wantRequests { - t.Errorf("upstream requests = %d, want %d", got, wantRequests) - } - }) - } -} - -func TestCondaRepodataBzip2StaysIdentity(t *testing.T) { - plain := []byte("BZh already compressed repodata") - for _, cache := range []bool{false, true} { - t.Run(map[bool]string{false: "stream", true: "cached"}[cache], func(t *testing.T) { - upstream := newGzipWhenAskedUpstream(plain, gzipPayload(t, plain)) - defer upstream.Close() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.CacheMetadata = cache - handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json.bz2", nil)) - if got := upstream.sawAcceptEncoding(); got != "identity" { - t.Errorf("upstream Accept-Encoding = %q, want identity", got) - } - if w.Code != http.StatusOK || !bytes.Equal(w.Body.Bytes(), plain) || w.Header().Get(headerContentEncoding) != "" { - t.Fatalf("unexpected response: %d, %v, %q", w.Code, w.Header(), w.Body.String()) - } - }) - } -} - -func TestCondaRepodataCacheSizeLimit(t *testing.T) { - plain := []byte(`{"packages":{},"padding":"` + strings.Repeat("x", 8192) + `"}`) - compressed := gzipPayload(t, plain) - for _, tt := range []struct { - name string - body []byte - encoding string - limit int64 - status int - }{ - {"identity fallback", plain, "", int64(len(plain)), http.StatusOK}, - {"identity too large", plain, "", int64(len(plain) - 1), http.StatusBadGateway}, - {"gzip too large", compressed, "gzip", int64(len(compressed) - 1), http.StatusBadGateway}, - } { - t.Run(tt.name, func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.Header.Get(headerAcceptEncoding); got != "gzip" { - t.Errorf("upstream Accept-Encoding = %q, want gzip", got) - } - w.Header().Set(headerContentEncoding, tt.encoding) - _, _ = w.Write(tt.body) - })) - defer upstream.Close() - proxy, db, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.CacheMetadata = true - proxy.MetadataMaxSize = tt.limit - h := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL) - handler := h.Routes() - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) - if w.Code != tt.status { - t.Fatalf("status = %d, want %d: %s", w.Code, tt.status, w.Body.String()) - } - if tt.status == http.StatusOK { - if !bytes.Equal(w.Body.Bytes(), plain) || w.Header().Get(headerContentEncoding) != "" { - t.Fatal("identity response was changed or mislabeled as gzip") - } - } else if entry, _ := db.GetMetadataCache("conda", h.metadataCacheKey("/conda-forge/linux-64/repodata.json")); entry != nil { - t.Fatal("oversized response was cached") - } - }) - } -} - -func TestCondaRepodataCooldownEncoding(t *testing.T) { - plain, err := json.Marshal(map[string]any{ - "packages": map[string]any{ - "old.tar.bz2": map[string]any{"name": "demo", "timestamp": time.Now().Add(-7 * 24 * time.Hour).UnixMilli()}, - "new.tar.bz2": map[string]any{"name": "demo", "timestamp": time.Now().UnixMilli()}, - }, - "packages.conda": map[string]any{ - "old.conda": map[string]any{"name": "demo", "timestamp": time.Now().Add(-7 * 24 * time.Hour).UnixMilli()}, - "new.conda": map[string]any{"name": "demo", "timestamp": time.Now().UnixMilli()}, - }, - }) - if err != nil { - t.Fatal(err) - } - for _, encoding := range []string{"identity", "gzip"} { - t.Run(encoding, func(t *testing.T) { - body := plain - if encoding == "gzip" { - body = gzipPayload(t, plain) - } - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(headerContentEncoding, encoding) - _, _ = w.Write(body) - })) - defer upstream.Close() - proxy := testProxy() - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() - for _, filename := range []string{"repodata.json", "current_repodata.json"} { - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+filename, nil)) - if w.Code != http.StatusOK || w.Header().Get(headerContentEncoding) != "" { - t.Fatalf("unexpected response: %d, %v", w.Code, w.Header()) - } - var result map[string]map[string]any - if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { - t.Fatal(err) - } - for key, old := range map[string]string{"packages": "old.tar.bz2", "packages.conda": "old.conda"} { - if len(result[key]) != 1 || result[key][old] == nil { - t.Errorf("%s: cooldown did not retain only the old package: %s", key, w.Body.String()) - } - } - } - }) - } -} - -func TestCondaRepodataCooldownRejectsInvalidGzip(t *testing.T) { - compressed := gzipPayload(t, []byte(`{"padding":"`+strings.Repeat("x", 8192)+`"}`)) - for _, tt := range []struct { - name string - body []byte - }{ - {"invalid header", []byte("not gzip")}, - {"truncated body", compressed[:len(compressed)-4]}, - } { - t.Run(tt.name, func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(headerContentEncoding, "gzip") - _, _ = w.Write(tt.body) - })) - defer upstream.Close() - proxy := testProxy() - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) - if w.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want 502", w.Code) - } - }) - } -} - -// largeRepodata builds repodata with n old and n new entries in each of -// packages and packages.conda. -func largeRepodata(n int) []byte { - old := time.Now().Add(-30 * 24 * time.Hour).UnixMilli() - recent := time.Now().UnixMilli() - var b strings.Builder - b.WriteString(`{"info":{"subdir":"linux-64"},"repodata_version":1`) - for _, key := range []string{"packages", "packages.conda"} { - fmt.Fprintf(&b, `,%q:{`, key) - for i := range 2 * n { - if i > 0 { - b.WriteByte(',') - } - ts, kind := old, "old" - if i%2 == 1 { - ts, kind = recent, "new" - } - fmt.Fprintf(&b, `"pkg-%d-%s.conda":{"name":"pkg%d","version":"1.0","timestamp":%d,"depends":["python >=3.10,<4","libgcc >=13"],"md5":"%032d"}`, - i, kind, i, ts, i) - } - b.WriteByte('}') - } - b.WriteString(`,"removed":[]}`) - return []byte(b.String()) -} - -func TestCondaRepodataCooldownStreamsPastMetadataLimit(t *testing.T) { - plain := largeRepodata(2000) - compressed := gzipPayload(t, plain) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(headerContentEncoding, "gzip") - _, _ = w.Write(compressed) - })) - defer upstream.Close() - - proxy := testProxy() - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - proxy.MetadataMaxSize = int64(len(plain) / 10) - handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() - - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200 for repodata %d times the metadata limit", w.Code, len(plain)/int(proxy.MetadataMaxSize)) - } - var result struct { - Info map[string]any `json:"info"` - Packages map[string]json.RawMessage `json:"packages"` - Conda map[string]json.RawMessage `json:"packages.conda"` - Removed []string `json:"removed"` - } - if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { - t.Fatalf("response is not valid JSON: %v", err) - } - if len(result.Packages) != 2000 || len(result.Conda) != 2000 || result.Info["subdir"] != "linux-64" || result.Removed == nil { - t.Fatalf("got %d packages, %d conda packages, info %v", len(result.Packages), len(result.Conda), result.Info) - } - for name := range result.Packages { - if strings.HasSuffix(name, "-new.conda") { - t.Fatalf("%s is inside the cooldown but was kept", name) - } - } - want := `"pkg-0-old.conda":{"name":"pkg0","version":"1.0","timestamp":` - if !strings.Contains(w.Body.String(), want) { - t.Errorf("kept entries should keep their upstream bytes") - } -} - -func TestCondaRepodataCooldownAbortsTruncatedStream(t *testing.T) { - compressed := gzipPayload(t, largeRepodata(2000)) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(headerContentEncoding, "gzip") - _, _ = w.Write(compressed[:len(compressed)*3/4]) - })) - defer upstream.Close() - - proxy := testProxy() - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - server := httptest.NewServer(NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes()) - defer server.Close() - - resp, err := server.Client().Get(server.URL + "/conda-forge/linux-64/repodata.json") - if err != nil { - t.Fatal(err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d; want the stream to start before the upstream body breaks", resp.StatusCode) - } - if _, err := io.ReadAll(resp.Body); err == nil { - t.Fatal("read a truncated repodata response without an error") - } -} diff --git a/internal/handler/conda_repodata_bench_test.go b/internal/handler/conda_repodata_bench_test.go deleted file mode 100644 index 61e68bd..0000000 --- a/internal/handler/conda_repodata_bench_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package handler - -import ( - "bytes" - "io" - "log/slog" - "os" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// repodataBenchBody reads the repodata.json named by CONDA_REPODATA_BENCH, -// such as conda-forge's noarch one, and skips the benchmark without it. -func repodataBenchBody(b *testing.B) []byte { - b.Helper() - path := os.Getenv("CONDA_REPODATA_BENCH") - if path == "" { - b.Skip("CONDA_REPODATA_BENCH names no repodata.json") - } - body, err := os.ReadFile(path) - if err != nil { - b.Fatal(err) - } - return body -} - -func BenchmarkCondaFilterRepodata(b *testing.B) { - body := repodataBenchBody(b) - policy, err := cooldown.NewPolicy(&cooldown.Config{Default: "3d"}, nil) - if err != nil { - b.Fatal(err) - } - proxy := testProxy() - proxy.Logger = slog.New(slog.DiscardHandler) - proxy.Cooldown = policy - h := NewCondaHandler(proxy, "http://proxy.example") - now := time.Now() - b.SetBytes(int64(len(body))) - b.ReportAllocs() - for b.Loop() { - if err := h.filterRepodataWith(bytes.NewReader(body), io.Discard, h.newRepodataFilter(now)); err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkCondaReadRepodataIndex(b *testing.B) { - body := repodataBenchBody(b) - b.SetBytes(int64(len(body))) - b.ReportAllocs() - for b.Loop() { - index, err := readRepodataIndex(bytes.NewReader(body)) - if err != nil || len(index) == 0 { - b.Fatalf("index of %d files, %v", len(index), err) - } - } -} diff --git a/internal/handler/conda_repodata_variants_test.go b/internal/handler/conda_repodata_variants_test.go deleted file mode 100644 index 48f997b..0000000 --- a/internal/handler/conda_repodata_variants_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package handler - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// The proxy filters only the plain JSON repodata. With cooldown on, any -// other form of it would list versions still in cooldown, so it is refused -// and clients fall back to repodata.json. -func TestCondaRepodataVariantsUnderCooldown(t *testing.T) { - plain, err := json.Marshal(map[string]any{ - "packages": map[string]any{ - "old.tar.bz2": map[string]any{"name": "demo", "timestamp": time.Now().Add(-7 * 24 * time.Hour).UnixMilli()}, - "new.tar.bz2": map[string]any{"name": "demo", "timestamp": time.Now().UnixMilli()}, - }, - }) - if err != nil { - t.Fatal(err) - } - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - _, _ = w.Write(plain) - })) - defer upstream.Close() - - variants := []string{ - "repodata.json.bz2", - "repodata.json.zst", - "repodata.jlap", - "current_repodata.json.zst", - "repodata_from_packages.json.zst", - } - for _, enabled := range []bool{true, false} { - proxy := testProxy() - proxy.HTTPClient = upstream.Client() - if enabled { - proxy.Cooldown = &cooldown.Config{Default: "3d"} - } - handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() - for _, filename := range variants { - requests.Store(0) - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+filename, nil)) - switch { - case enabled && (w.Code != http.StatusNotFound || requests.Load() != 0): - t.Errorf("cooldown on, %s: status %d, %d upstream requests; want 404 without asking upstream", filename, w.Code, requests.Load()) - case !enabled && (w.Code != http.StatusOK || requests.Load() != 1): - t.Errorf("cooldown off, %s: status %d, %d upstream requests; want it passed through", filename, w.Code, requests.Load()) - } - } - } - - // repodata_from_packages.json is plain JSON and is filtered like - // repodata.json. - proxy := testProxy() - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata_from_packages.json", nil)) - var result map[string]map[string]any - if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { - t.Fatalf("status %d, body %s: %v", w.Code, w.Body, err) - } - if len(result["packages"]) != 1 || result["packages"]["old.tar.bz2"] == nil { - t.Errorf("repodata_from_packages.json not filtered: %s", w.Body) - } -} diff --git a/internal/handler/conda_shards.go b/internal/handler/conda_shards.go deleted file mode 100644 index 8269f1d..0000000 --- a/internal/handler/conda_shards.go +++ /dev/null @@ -1,457 +0,0 @@ -package handler - -import ( - "bytes" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "net/http" - "net/url" - "path" - "strconv" - "strings" - "sync" - "time" - - "github.com/klauspost/compress/zstd" - "github.com/vmihailenco/msgpack/v5" -) - -// Sharded repodata (CEP 16) maps each package name to the sha256 of its -// shard. With cooldown the proxy filters shards, which changes their bytes, -// but clients cache shards by hash and never fetch the same hash again. So -// the rewritten index XORs every hash with a mask that changes each hour: -// filtered shards get hashes of their own, and clients fetch them again once -// versions have left cooldown. -const ( - condaShardIndexFile = "repodata_shards.msgpack.zst" - condaShardSuffix = ".msgpack.zst" - condaCooldownShardDir = "cooldown-shards" - condaShardsSubdir = "shards/" - condaShardBucket = time.Hour - condaShardIndexFormat = 1 - condaShardHashLen = sha256.Size - // maxCondaShardSize bounds a decompressed index or shard. conda-forge's - // largest index is a few MB. - maxCondaShardSize = 256 << 20 -) - -var ( - errCondaShardLayout = errors.New("unsupported sharded repodata layout") - - condaZstd = sync.OnceValues(func() (*zstd.Decoder, error) { - return zstd.NewReader(nil, zstd.WithDecoderMaxMemory(maxCondaShardSize), zstd.WithDecoderConcurrency(0)) - }) - condaZstdEncoder = sync.OnceValues(func() (*zstd.Encoder, error) { - return zstd.NewWriter(nil) - }) -) - -func (h *CondaHandler) cooldownEnabled() bool { - return h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() -} - -// handleShardIndex serves repodata_shards.msgpack.zst. -func (h *CondaHandler) handleShardIndex(w http.ResponseWriter, r *http.Request) { - channel, arch := r.PathValue("channel"), r.PathValue("arch") - upstreamURL := h.upstreamURL + r.URL.Path - cacheKey := h.metadataCacheKey(r.URL.Path) - - body, contentType, contentEncoding, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", cacheKey, upstreamURL, "identity", nil, "*/*") - if err != nil { - h.proxy.writeMetadataFetchError(w, err) - return - } - - if !h.cooldownEnabled() { - index, err := decodeCondaShardIndex(body) - if err == nil { - _, err = index.shardsLayout(upstreamURL) - } - if err != nil { - h.proxy.Logger.Info("serving no sharded repodata", "url", upstreamURL, "error", err) - http.NotFound(w, r) - return - } - h.proxy.writeMetadataCachedResponseWithEncoding(w, r, "conda", cacheKey, body, contentType, contentEncoding) - return - } - - now := time.Now() - bucket := strconv.FormatInt(now.Unix()/int64(condaShardBucket/time.Second), 10) - rewritten, err := rewriteCondaShardIndex(body, upstreamURL, condaShardMask(channel, arch, bucket), bucket) - if err != nil { - h.proxy.Logger.Info("serving no sharded repodata", "url", upstreamURL, "error", err) - http.NotFound(w, r) - return - } - - next := now.Truncate(condaShardBucket).Add(condaShardBucket) - maxAge := max(int(next.Sub(now)/time.Second), 1) - w.Header().Set(headerContentType, "application/octet-stream") - w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge)) - _, _ = w.Write(rewritten) -} - -// handleShard serves a shard under its upstream hash. While cooldown is on, -// the index never points here, and the unfiltered shard is refused. -func (h *CondaHandler) handleShard(w http.ResponseWriter, r *http.Request) { - if h.cooldownEnabled() || !isCondaShardFile(path.Base(r.URL.Path)) { - http.NotFound(w, r) - return - } - h.proxyCached(w, r) -} - -// handleCooldownShard serves a shard named in a rewritten index, filtered by -// cooldown. -func (h *CondaHandler) handleCooldownShard(w http.ResponseWriter, r *http.Request) { - channel, arch, bucket := r.PathValue("channel"), r.PathValue("arch"), r.PathValue("bucket") - dir, file := path.Split(r.PathValue("rest")) - if _, err := strconv.ParseUint(bucket, 10, 64); err != nil || (dir != "" && dir != condaShardsSubdir) || !isCondaShardFile(file) { - http.NotFound(w, r) - return - } - - masked, _ := hex.DecodeString(strings.TrimSuffix(file, condaShardSuffix)) - mask := condaShardMask(channel, arch, bucket) - for i := range masked { - masked[i] ^= mask[i] - } - upstreamPath := "/" + channel + "/" + arch + "/" + dir + hex.EncodeToString(masked) + condaShardSuffix - - body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", h.metadataCacheKey(upstreamPath), h.upstreamURL+upstreamPath, "identity", nil, "*/*") - if err != nil { - h.proxy.writeMetadataFetchError(w, err) - return - } - - if h.cooldownEnabled() { - body, err = h.filterCondaShard(body) - if err != nil { - h.proxy.Logger.Error("failed to filter conda shard", "path", upstreamPath, "error", err) - http.Error(w, "failed to filter shard", http.StatusBadGateway) - return - } - } - w.Header().Set(headerContentType, "application/octet-stream") - _, _ = w.Write(body) -} - -func isCondaShardFile(name string) bool { - hash, ok := strings.CutSuffix(name, condaShardSuffix) - if !ok || len(hash) != 2*condaShardHashLen { - return false - } - _, err := hex.DecodeString(hash) - return err == nil && strings.ToLower(hash) == hash -} - -// condaShardMask is XORed with shard hashes in a rewritten index. It keeps -// filtered shards apart from upstream ones in client caches, which are keyed -// by hash alone. -func condaShardMask(channel, arch, bucket string) [condaShardHashLen]byte { - return sha256.Sum256([]byte("git-pkgs proxy conda cooldown shard\x00" + channel + "\x00" + arch + "\x00" + bucket)) -} - -// msgpackEntry is one member of a msgpack map, kept in its upstream order -// with its value undecoded. -type msgpackEntry struct { - key string - value msgpack.RawMessage -} - -func decodeMsgpackMap(b []byte) ([]msgpackEntry, error) { - dec := msgpack.NewDecoder(bytes.NewReader(b)) - n, err := dec.DecodeMapLen() - if err != nil { - return nil, err - } - if n < 0 { - return nil, errors.New("expected a msgpack map") - } - entries := make([]msgpackEntry, 0, n) - for range n { - key, err := dec.DecodeString() - if err != nil { - return nil, err - } - value, err := dec.DecodeRaw() - if err != nil { - return nil, err - } - entries = append(entries, msgpackEntry{key: key, value: value}) - } - return entries, nil -} - -func encodeMsgpackMap(entries []msgpackEntry) ([]byte, error) { - var buf bytes.Buffer - enc := msgpack.NewEncoder(&buf) - if err := enc.EncodeMapLen(len(entries)); err != nil { - return nil, err - } - for _, e := range entries { - if err := enc.EncodeString(e.key); err != nil { - return nil, err - } - if err := enc.Encode(e.value); err != nil { - return nil, err - } - } - return buf.Bytes(), nil -} - -func lookupMsgpackEntry(entries []msgpackEntry, key string) (int, bool) { - for i, e := range entries { - if e.key == key { - return i, true - } - } - return -1, false -} - -func decompressCondaShard(body []byte) ([]byte, error) { - dec, err := condaZstd() - if err != nil { - return nil, err - } - return dec.DecodeAll(body, nil) -} - -func compressCondaShard(body []byte) ([]byte, error) { - enc, err := condaZstdEncoder() - if err != nil { - return nil, err - } - return enc.EncodeAll(body, nil), nil -} - -// condaShardIndex is a decoded repodata_shards.msgpack.zst. -type condaShardIndex struct { - top []msgpackEntry - info []msgpackEntry -} - -func decodeCondaShardIndex(body []byte) (*condaShardIndex, error) { - raw, err := decompressCondaShard(body) - if err != nil { - return nil, err - } - top, err := decodeMsgpackMap(raw) - if err != nil { - return nil, err - } - i, ok := lookupMsgpackEntry(top, "version") - if !ok { - return nil, errors.New("sharded index has no version") - } - var version int - if err := msgpack.Unmarshal(top[i].value, &version); err != nil || version != condaShardIndexFormat { - return nil, fmt.Errorf("unsupported sharded index version %v", top[i].value) - } - i, ok = lookupMsgpackEntry(top, "info") - if !ok { - return nil, errors.New("sharded index has no info") - } - info, err := decodeMsgpackMap(top[i].value) - if err != nil { - return nil, err - } - return &condaShardIndex{top: top, info: info}, nil -} - -func (idx *condaShardIndex) infoString(key string) (string, error) { - i, ok := lookupMsgpackEntry(idx.info, key) - if !ok { - return "", nil - } - var s string - err := msgpack.Unmarshal(idx.info[i].value, &s) - return s, err -} - -// shardsLayout checks that packages and shards sit where the proxy routes -// them, below the subdir of indexURL, and returns the shards directory -// relative to the subdir: "" or "shards/". Clients resolve base_url and -// shards_base_url against the proxy's URL, so only relative paths stay on -// the proxy: a scheme or host leads clients around it, a leading / outside -// its mount. -func (idx *condaShardIndex) shardsLayout(indexURL string) (string, error) { - base, err := url.Parse(indexURL) - if err != nil { - return "", err - } - subdir := base.ResolveReference(&url.URL{Path: "./"}).String() - resolve := func(ref string) (string, error) { - if ref == "" { - return subdir, nil - } - u, err := url.Parse(ref) - if err != nil { - return "", err - } - if u.Scheme != "" || u.Host != "" || strings.HasPrefix(u.Path, "/") { - return "", errCondaShardLayout - } - return base.ResolveReference(u).String(), nil - } - - baseURL, err := idx.infoString("base_url") - if err != nil { - return "", err - } - if resolved, err := resolve(baseURL); err != nil || resolved != subdir { - return "", fmt.Errorf("%w: base_url %q", errCondaShardLayout, baseURL) - } - - shardsBaseURL, err := idx.infoString("shards_base_url") - if err != nil { - return "", err - } - resolved, err := resolve(shardsBaseURL) - rel, ok := strings.CutPrefix(resolved, subdir) - if err != nil || !ok || (rel != "" && rel != condaShardsSubdir) { - return "", fmt.Errorf("%w: shards_base_url %q", errCondaShardLayout, shardsBaseURL) - } - return rel, nil -} - -// rewriteCondaShardIndex masks every shard hash and points shards_base_url -// at the cooldown shard route for bucket. -func rewriteCondaShardIndex(body []byte, indexURL string, mask [condaShardHashLen]byte, bucket string) ([]byte, error) { - idx, err := decodeCondaShardIndex(body) - if err != nil { - return nil, err - } - layout, err := idx.shardsLayout(indexURL) - if err != nil { - return nil, err - } - - shardsBase, err := msgpack.Marshal("./" + condaCooldownShardDir + "/" + bucket + "/" + layout) - if err != nil { - return nil, err - } - if i, ok := lookupMsgpackEntry(idx.info, "shards_base_url"); ok { - idx.info[i].value = shardsBase - } else { - idx.info = append(idx.info, msgpackEntry{key: "shards_base_url", value: shardsBase}) - } - info, err := encodeMsgpackMap(idx.info) - if err != nil { - return nil, err - } - i, _ := lookupMsgpackEntry(idx.top, "info") - idx.top[i].value = info - - i, ok := lookupMsgpackEntry(idx.top, "shards") - if !ok { - return nil, errors.New("sharded index has no shards") - } - shards, err := decodeMsgpackMap(idx.top[i].value) - if err != nil { - return nil, err - } - for j := range shards { - var hash []byte - if err := msgpack.Unmarshal(shards[j].value, &hash); err != nil || len(hash) != condaShardHashLen { - return nil, fmt.Errorf("shard hash for %q is not a sha256", shards[j].key) - } - for k := range hash { - hash[k] ^= mask[k] - } - if shards[j].value, err = msgpack.Marshal(hash); err != nil { - return nil, err - } - } - if idx.top[i].value, err = encodeMsgpackMap(shards); err != nil { - return nil, err - } - - raw, err := encodeMsgpackMap(idx.top) - if err != nil { - return nil, err - } - return compressCondaShard(raw) -} - -// condaShardRecord holds the fields cooldown needs from a shard record. -// Timestamps are an integer or a float, in seconds or milliseconds as -// condaTime reads them. -type condaShardRecord struct { - Name string `msgpack:"name"` - Version string `msgpack:"version"` - Timestamp any `msgpack:"timestamp"` -} - -// filterCondaShard leaves out records in packages and packages.conda that -// are still inside their cooldown. Kept records keep their upstream bytes. -func (h *CondaHandler) filterCondaShard(body []byte) ([]byte, error) { - raw, err := decompressCondaShard(body) - if err != nil { - return nil, err - } - top, err := decodeMsgpackMap(raw) - if err != nil { - return nil, err - } - for i := range top { - if top[i].key != "packages" && top[i].key != "packages.conda" { - continue - } - records, err := decodeMsgpackMap(top[i].value) - if err != nil { - return nil, err - } - kept := records[:0] - for _, rec := range records { - var fields condaShardRecord - if err := msgpack.Unmarshal(rec.value, &fields); err != nil { - return nil, fmt.Errorf("record %q: %w", rec.key, err) - } - if h.recordInCooldown(rec.key, fields.Name, fields.Version, msgpackNumber(fields.Timestamp)) { - continue - } - kept = append(kept, rec) - } - if top[i].value, err = encodeMsgpackMap(kept); err != nil { - return nil, err - } - } - out, err := encodeMsgpackMap(top) - if err != nil { - return nil, err - } - return compressCondaShard(out) -} - -// msgpackNumber returns a decoded msgpack integer or float as float64, and 0 -// for anything else. -func msgpackNumber(v any) float64 { - switch n := v.(type) { - case int8: - return float64(n) - case int16: - return float64(n) - case int32: - return float64(n) - case int64: - return float64(n) - case uint8: - return float64(n) - case uint16: - return float64(n) - case uint32: - return float64(n) - case uint64: - return float64(n) - case float32: - return float64(n) - case float64: - return n - default: - return 0 - } -} diff --git a/internal/handler/conda_shards_test.go b/internal/handler/conda_shards_test.go deleted file mode 100644 index 27b0ad7..0000000 --- a/internal/handler/conda_shards_test.go +++ /dev/null @@ -1,289 +0,0 @@ -package handler - -import ( - "bytes" - "crypto/sha256" - "encoding/hex" - "net/http" - "net/http/httptest" - "sort" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/vmihailenco/msgpack/v5" -) - -func zstdMsgpack(t *testing.T, v any) []byte { - t.Helper() - raw, err := msgpack.Marshal(v) - if err != nil { - t.Fatal(err) - } - out, err := compressCondaShard(raw) - if err != nil { - t.Fatal(err) - } - return out -} - -func decodeZstdMsgpack(t *testing.T, body []byte, v any) { - t.Helper() - raw, err := decompressCondaShard(body) - if err != nil { - t.Fatal(err) - } - if err := msgpack.Unmarshal(raw, v); err != nil { - t.Fatal(err) - } -} - -type shardFixture struct { - upstream *httptest.Server - shardHash string - requests []string -} - -// newShardUpstream serves a sharded index for conda-forge/noarch with one -// shard for testpkg, holding an old and a recent build. -func newShardUpstream(t *testing.T, info map[string]any) *shardFixture { - t.Helper() - now := time.Now() - shard := zstdMsgpack(t, map[string]any{ - "packages": map[string]any{ - "testpkg-1.0.0-0.tar.bz2": map[string]any{ - "name": "testpkg", "version": "1.0.0", "timestamp": now.Add(-7 * 24 * time.Hour).UnixMilli(), - "sha256": bytes.Repeat([]byte{1}, 32), - }, - }, - "packages.conda": map[string]any{ - "testpkg-1.0.0-0.conda": map[string]any{ - "name": "testpkg", "version": "1.0.0", "timestamp": now.Add(-7 * 24 * time.Hour).UnixMilli(), - }, - "testpkg-2.0.0-0.conda": map[string]any{ - "name": "testpkg", "version": "2.0.0", "timestamp": now.Add(-time.Hour).UnixMilli(), - }, - }, - "removed": []string{"testpkg-0.1.0-0.tar.bz2"}, - }) - sum := sha256.Sum256(shard) - f := &shardFixture{shardHash: hex.EncodeToString(sum[:])} - - layout := "" - if s, _ := info["shards_base_url"].(string); strings.Contains(s, "shards/") { - layout = "shards/" - } - var index []byte - f.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - f.requests = append(f.requests, r.URL.Path) - switch r.URL.Path { - case "/conda-forge/noarch/" + condaShardIndexFile: - _, _ = w.Write(index) - case "/conda-forge/noarch/" + layout + f.shardHash + condaShardSuffix: - _, _ = w.Write(shard) - default: - http.NotFound(w, r) - } - })) - t.Cleanup(f.upstream.Close) - - // {upstream} in an info value stands for the upstream's URL, {host} for - // its host. - placeholders := strings.NewReplacer("{upstream}", f.upstream.URL, "{host}", strings.TrimPrefix(f.upstream.URL, "http://")) - for k, v := range info { - if s, ok := v.(string); ok { - info[k] = placeholders.Replace(s) - } - } - index = zstdMsgpack(t, map[string]any{ - "version": 1, - "info": info, - "shards": map[string]any{"testpkg": sum[:]}, - }) - return f -} - -func shardTestHandler(f *shardFixture, cooldownDefault string) http.Handler { - proxy := testProxy() - if cooldownDefault != "" { - proxy.Cooldown = &cooldown.Config{Default: cooldownDefault} - } - return NewCondaHandlerWithUpstream(proxy, "http://proxy.local", f.upstream.URL).Routes() -} - -func serveShardRequest(h http.Handler, path string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - return w -} - -type shardRecords struct { - Packages map[string]map[string]any `msgpack:"packages"` - PackagesConda map[string]map[string]any `msgpack:"packages.conda"` - Removed []string `msgpack:"removed"` -} - -func (s shardRecords) filenames() []string { - var names []string - for k := range s.Packages { - names = append(names, k) - } - for k := range s.PackagesConda { - names = append(names, k) - } - sort.Strings(names) - return names -} - -type shardIndex struct { - Version int `msgpack:"version"` - Info map[string]any `msgpack:"info"` - Shards map[string][]byte `msgpack:"shards"` -} - -func TestCondaShardsWithoutCooldown(t *testing.T) { - for _, shardsBaseURL := range []string{"", "./shards/"} { - t.Run(shardsBaseURL, func(t *testing.T) { - f := newShardUpstream(t, map[string]any{"base_url": "", "shards_base_url": shardsBaseURL, "subdir": "noarch"}) - h := shardTestHandler(f, "") - - w := serveShardRequest(h, "/conda-forge/noarch/"+condaShardIndexFile) - if w.Code != http.StatusOK { - t.Fatalf("index status = %d", w.Code) - } - var index shardIndex - decodeZstdMsgpack(t, w.Body.Bytes(), &index) - if got := hex.EncodeToString(index.Shards["testpkg"]); got != f.shardHash { - t.Fatalf("index hash = %s, want upstream %s", got, f.shardHash) - } - if index.Info["shards_base_url"] != shardsBaseURL { - t.Fatalf("shards_base_url = %v, want %q", index.Info["shards_base_url"], shardsBaseURL) - } - - dir := strings.TrimPrefix(shardsBaseURL, "./") - w = serveShardRequest(h, "/conda-forge/noarch/"+dir+f.shardHash+condaShardSuffix) - if w.Code != http.StatusOK { - t.Fatalf("shard status = %d", w.Code) - } - var shard shardRecords - decodeZstdMsgpack(t, w.Body.Bytes(), &shard) - if len(shard.filenames()) != 3 { - t.Fatalf("records = %v, want all 3", shard.filenames()) - } - }) - } -} - -func TestCondaShardsWithCooldown(t *testing.T) { - f := newShardUpstream(t, map[string]any{"base_url": "", "shards_base_url": "./shards/", "subdir": "noarch", "created_at": "2026-01-01T00:00:00Z"}) - h := shardTestHandler(f, "3d") - - w := serveShardRequest(h, "/conda-forge/noarch/"+condaShardIndexFile) - if w.Code != http.StatusOK { - t.Fatalf("index status = %d", w.Code) - } - if !strings.HasPrefix(w.Header().Get("Cache-Control"), "public, max-age=") { - t.Errorf("Cache-Control = %q", w.Header().Get("Cache-Control")) - } - var index shardIndex - decodeZstdMsgpack(t, w.Body.Bytes(), &index) - masked := hex.EncodeToString(index.Shards["testpkg"]) - if masked == f.shardHash || len(masked) != 64 { - t.Fatalf("index hash = %s, want a masked hash", masked) - } - if index.Info["created_at"] != "2026-01-01T00:00:00Z" || index.Info["subdir"] != "noarch" { - t.Errorf("info lost fields: %v", index.Info) - } - shardsBaseURL, _ := index.Info["shards_base_url"].(string) - if !strings.HasPrefix(shardsBaseURL, "./"+condaCooldownShardDir+"/") || !strings.HasSuffix(shardsBaseURL, "/shards/") { - t.Fatalf("shards_base_url = %q", shardsBaseURL) - } - - w = serveShardRequest(h, "/conda-forge/noarch/"+strings.TrimPrefix(shardsBaseURL, "./")+masked+condaShardSuffix) - if w.Code != http.StatusOK { - t.Fatalf("shard status = %d: %s", w.Code, w.Body) - } - var shard shardRecords - decodeZstdMsgpack(t, w.Body.Bytes(), &shard) - want := []string{"testpkg-1.0.0-0.conda", "testpkg-1.0.0-0.tar.bz2"} - if got := shard.filenames(); strings.Join(got, ",") != strings.Join(want, ",") { - t.Fatalf("records = %v, want %v", got, want) - } - if len(shard.Removed) != 1 { - t.Errorf("removed = %v, want it kept", shard.Removed) - } - if _, ok := shard.Packages["testpkg-1.0.0-0.tar.bz2"]["sha256"].([]byte); !ok { - t.Errorf("sha256 is no longer binary: %T", shard.Packages["testpkg-1.0.0-0.tar.bz2"]["sha256"]) - } - - // The unfiltered shard stays out of reach while cooldown is on. - w = serveShardRequest(h, "/conda-forge/noarch/shards/"+f.shardHash+condaShardSuffix) - if w.Code != http.StatusNotFound { - t.Fatalf("plain shard status = %d, want 404", w.Code) - } -} - -func TestCondaCooldownShardRejectsBadPaths(t *testing.T) { - f := newShardUpstream(t, map[string]any{"base_url": "", "shards_base_url": ""}) - h := shardTestHandler(f, "3d") - hash := strings.Repeat("a", 64) + condaShardSuffix - for _, p := range []string{ - "/conda-forge/noarch/cooldown-shards/abc/" + hash, - "/conda-forge/noarch/cooldown-shards/1/other/" + hash, - "/conda-forge/noarch/cooldown-shards/1/" + strings.Repeat("A", 64) + condaShardSuffix, - "/conda-forge/noarch/cooldown-shards/1/x.msgpack.zst", - } { - if w := serveShardRequest(h, p); w.Code != http.StatusNotFound { - t.Errorf("%s: status = %d, want 404", p, w.Code) - } - } - if len(f.requests) != 0 { - t.Errorf("upstream was asked for %v", f.requests) - } -} - -func TestCondaShardIndexUnsupportedLayout(t *testing.T) { - for name, info := range map[string]map[string]any{ - "packages elsewhere": {"base_url": "https://cdn.example.com/conda-forge/noarch/", "shards_base_url": ""}, - "shards elsewhere": {"base_url": "", "shards_base_url": "https://cdn.example.com/shards/"}, - "nested shards": {"base_url": "", "shards_base_url": "./a/b/"}, - "packages upstream": {"base_url": "{upstream}/conda-forge/noarch/", "shards_base_url": ""}, - "shards upstream": {"base_url": "", "shards_base_url": "{upstream}/conda-forge/noarch/"}, - "no scheme upstream": {"base_url": "//{host}/conda-forge/noarch/", "shards_base_url": ""}, - "root-relative": {"base_url": "/conda-forge/noarch/", "shards_base_url": ""}, - "root-relative shards": {"base_url": "", "shards_base_url": "/conda-forge/noarch/shards/"}, - } { - t.Run(name, func(t *testing.T) { - f := newShardUpstream(t, info) - for _, cd := range []string{"", "3d"} { - w := serveShardRequest(shardTestHandler(f, cd), "/conda-forge/noarch/"+condaShardIndexFile) - if w.Code != http.StatusNotFound { - t.Errorf("cooldown %q: status = %d, want 404 so clients use repodata.json", cd, w.Code) - } - } - }) - } -} - -func TestCondaShardIndexUnknownVersion(t *testing.T) { - index := zstdMsgpack(t, map[string]any{"version": 2, "info": map[string]any{}, "shards": map[string]any{}}) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write(index) - })) - defer upstream.Close() - - proxy := testProxy() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - h := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() - if w := serveShardRequest(h, "/conda-forge/noarch/"+condaShardIndexFile); w.Code != http.StatusNotFound { - t.Fatalf("status = %d, want 404", w.Code) - } -} - -func TestCondaShardMaskDependsOnBucket(t *testing.T) { - a := condaShardMask("conda-forge", "noarch", "1") - if a == condaShardMask("conda-forge", "noarch", "2") || a == condaShardMask("conda-forge", "linux-64", "1") { - t.Fatal("mask must change with bucket and subdir") - } -} diff --git a/internal/handler/conda_subdir_test.go b/internal/handler/conda_subdir_test.go deleted file mode 100644 index 1a81e48..0000000 --- a/internal/handler/conda_subdir_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package handler - -import ( - "net/http" - "net/http/httptest" - "sync/atomic" - "testing" - - "github.com/git-pkgs/registries/fetch" -) - -// The same conda filename can hold different bytes in each subdir, so each -// subdir must get its own cache entry. -func TestCondaDownloadCachesPerSubdir(t *testing.T) { - const filename = "python_abi-3.11-4_cp311.conda" - packages := map[string]string{ - "/conda-forge/linux-64/" + filename: "linux-64 bytes", - "/conda-forge/osx-arm64/" + filename: "osx-arm64 bytes", - } - - var available atomic.Bool - available.Store(true) - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - data, ok := packages[r.URL.Path] - if !available.Load() || !ok { - http.NotFound(w, r) - return - } - requests.Add(1) - _, _ = w.Write([]byte(data)) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - routes := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() - - get := func(path string) string { - w := httptest.NewRecorder() - routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - if w.Code != http.StatusOK { - t.Fatalf("%s: status = %d: %s", path, w.Code, w.Body.String()) - } - return w.Body.String() - } - - for _, round := range []string{"fetch", "cached"} { - for path, want := range packages { - if got := get(path); got != want { - t.Errorf("%s %s: body = %q, want %q", round, path, got, want) - } - } - available.Store(false) - } - if got := requests.Load(); got != 2 { - t.Errorf("upstream requests = %d, want 2 (one per subdir)", got) - } -} diff --git a/internal/handler/conda_test.go b/internal/handler/conda_test.go index f6c37c6..1b57039 100644 --- a/internal/handler/conda_test.go +++ b/internal/handler/conda_test.go @@ -1,20 +1,14 @@ package handler import ( - "bytes" "encoding/json" - "fmt" "log/slog" - "maps" "net/http" "net/http/httptest" - "slices" - "strings" "testing" "time" "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/purl" ) func TestCondaParseFilename(t *testing.T) { @@ -110,7 +104,7 @@ func TestCondaCooldownFiltering(t *testing.T) { proxyURL: "http://localhost:8080", } - filtered, err := filterRepodataBytes(h, body) + filtered, err := h.applyCooldownFiltering(body) if err != nil { t.Fatal(err) } @@ -137,45 +131,6 @@ func TestCondaCooldownFiltering(t *testing.T) { } } -// TestCondaCooldownSecondTimestamps checks that a timestamp in seconds, -// which some repodata entries carry, is read as seconds, as conda does, in -// both the filter and the download index. -func TestCondaCooldownSecondTimestamps(t *testing.T) { - recent := time.Now().Add(-time.Hour) - body := fmt.Sprintf(`{"packages":{"seconds-1.0-0.tar.bz2":{"name":"seconds","version":"1.0","timestamp":%d},`+ - `"millis-1.0-0.tar.bz2":{"name":"millis","version":"1.0","timestamp":%d}}}`, recent.Unix(), recent.UnixMilli()) - - proxy := testProxy() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - h := &CondaHandler{proxy: proxy, proxyURL: "http://localhost:8080"} - filtered, err := filterRepodataBytes(h, []byte(body)) - if err != nil { - t.Fatal(err) - } - var result struct { - Packages map[string]json.RawMessage `json:"packages"` - } - if err := json.Unmarshal(filtered, &result); err != nil { - t.Fatal(err) - } - if len(result.Packages) != 0 { - t.Errorf("packages = %v, want both left out", slices.Sorted(maps.Keys(result.Packages))) - } - - index, err := readRepodataIndex(strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } - if len(index) != 2 { - t.Fatalf("index has %d entries, want 2", len(index)) - } - for filename, published := range index { - if published.Unix() != recent.Unix() { - t.Errorf("%s published at %v, want %v", filename, published, recent) - } - } -} - func TestCondaCooldownFilteringWithPackageOverride(t *testing.T) { now := time.Now() recentTimestamp := float64(now.Add(-2 * time.Hour).UnixMilli()) @@ -208,7 +163,7 @@ func TestCondaCooldownFilteringWithPackageOverride(t *testing.T) { proxyURL: "http://localhost:8080", } - filtered, err := filterRepodataBytes(h, body) + filtered, err := h.applyCooldownFiltering(body) if err != nil { t.Fatal(err) } @@ -252,7 +207,7 @@ func TestCondaCooldownFilteringNoTimestamp(t *testing.T) { proxyURL: "http://localhost:8080", } - filtered, err := filterRepodataBytes(h, body) + filtered, err := h.applyCooldownFiltering(body) if err != nil { t.Fatal(err) } @@ -348,41 +303,3 @@ func TestCondaHandleRepodataWithoutCooldown(t *testing.T) { t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } } - -func TestCondaRetentionKey(t *testing.T) { - tests := []struct { - key string - want string - ok bool - }{ - {"pkg:conda/conda-forge/numpy", "pkg:conda/conda-forge%2Fnumpy", true}, - {"pkg:conda/conda-forge%2Fnumpy", "pkg:conda/conda-forge%2Fnumpy", true}, - {"pkg:conda/conda-forge/PyYAML", "pkg:conda/conda-forge%2Fpyyaml", true}, - {"pkg:conda/Conda-Forge/numpy", "pkg:conda/Conda-Forge%2Fnumpy", true}, - {"pkg:conda/conda-forge/label/rc/numpy", "", false}, - {"pkg:conda/conda-forge%2Flabel%2Frc%2Fnumpy", "", false}, - {"pkg:conda/numpy", "", false}, - {"pkg:conda/%2Fnumpy", "", false}, - {"pkg:conda/conda-forge%2F", "", false}, - {"pkg:npm/conda-forge/numpy", "", false}, - } - for _, tt := range tests { - p, err := purl.Parse(tt.key) - if err != nil { - t.Fatalf("Parse(%s): %v", tt.key, err) - } - got, ok := condaRetentionKey(p) - if ok != tt.ok || (ok && got != tt.want) { - t.Errorf("condaRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) - } - } - if _, ok := condaRetentionKey(nil); ok { - t.Error("condaRetentionKey(nil) accepted") - } -} - -func filterRepodataBytes(h *CondaHandler, body []byte) ([]byte, error) { - var out bytes.Buffer - err := h.filterRepodata(bytes.NewReader(body), &out) - return out.Bytes(), err -} diff --git a/internal/handler/container.go b/internal/handler/container.go index 40f5d0f..8ba5e97 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -2,341 +2,39 @@ package handler import ( "encoding/json" - "errors" "fmt" - "maps" - "net" + "io" "net/http" - "net/url" "regexp" - "slices" "strings" - "sync" - - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/purl" - "golang.org/x/sync/singleflight" ) const ( - dockerHubRegistry = "https://registry-1.docker.io" - blobMatchCount = 3 // full match + name + digest - manifestMatchCount = 3 // full match + name + reference - tagsListMatchCount = 2 // full match + name - registrySelectorParts = 3 // upstream + name + repository - - // namespaceQueryParam is the query parameter containerd appends to mirror - // requests to name the registry the image reference points at. - namespaceQueryParam = "ns" - // defaultNamespaceRoute marks namespace hosts served by the default registry. - defaultNamespaceRoute = "" - // maxReportedNamespaceRefusals bounds the upstream/ns pairs remembered - // for logging refusals once; past it further refusals are not logged. - maxReportedNamespaceRefusals = 1024 - // maxNamespaceLength is the longest ns a registry host can produce: a - // 253-character DNS name plus ":65535". Longer values are refused before - // they are logged or remembered. - maxNamespaceLength = 259 + dockerHubRegistry = "https://registry-1.docker.io" + dockerHubAuth = "https://auth.docker.io" + blobMatchCount = 3 // full match + name + digest + manifestMatchCount = 3 // full match + name + reference + tagsListMatchCount = 2 // full match + name ) -// dockerHubNamespaces are the registry URLs clients use for Docker Hub. -var dockerHubNamespaces = []string{"https://docker.io", "https://index.docker.io", "https://registry-1.docker.io"} //nolint:gochecknoglobals // fixed alias list - -// schemeDefaultPorts are the ports an image reference may leave out. -var schemeDefaultPorts = map[string]string{"https": "443", "http": "80"} //nolint:gochecknoglobals // fixed table - -// dockerHubNamespaceKeys are the ns lookup keys of the Docker Hub aliases. -var dockerHubNamespaceKeys = dockerHubKeys() //nolint:gochecknoglobals // fixed table - -func dockerHubKeys() []string { - var keys []string - for _, registryURL := range dockerHubNamespaces { - hostKeys, _ := namespaceKeysForURL(registryURL) - keys = append(keys, hostKeys...) - } - return keys -} - -func isDockerHubKey(key string) bool { - return slices.Contains(dockerHubNamespaceKeys, key) -} - -// Blobs are stored under the repository name the client asks for, which -// purl writes as one name, pkg:oci/library%2Fnginx. Named upstreams keep -// their upstream/{name}/ prefix, also when containerd reaches them through -// ns. Homebrew bottles are stored under Homebrew's image name, which writes -// openssl@3 as homebrew/core/openssl/3, and Helm charts pulled over OCI go -// through here like images. -// Package rules write the repository name as a path, pkg:oci/library/nginx, -// and are mapped to that form. -func init() { - retention.Default.Register(retention.Spec{Key: "oci", CanonicalPackage: ociRetentionKey}) -} - -// ociRetentionKey maps a package rule to the stored package PURL. It refuses -// a key with an empty path segment, a segment with ':' or '@' (a tag or -// digest from an image reference), or an upstream/{name} prefix without a -// repository, since no blob is ever stored under such a name. -func ociRetentionKey(p *purl.PURL) (string, bool) { - if p == nil { - return "", false - } - name := p.Name - if p.Namespace != "" { - name = p.Namespace + "/" + p.Name - } - segments := strings.Split(name, "/") - if slices.Contains(segments, "") || strings.ContainsAny(name, ":@") || - (segments[0] == "upstream" && len(segments) < registrySelectorParts) { - return "", false - } - return retention.DefaultCanonical("oci")(purl.New(p.Type, "", name, "", nil)) -} - // ContainerHandler handles OCI/Docker container registry protocol requests. // It implements the OCI Distribution Spec for pulling images. // Reference: https://github.com/opencontainers/distribution-spec/blob/main/spec.md type ContainerHandler struct { - proxy *Proxy - registryURL string - proxyURL string - namedRegistries map[string]string - registries []containerRegistry - // namespaces maps normalized registry hosts from containerd's ns query - // parameter to a named upstream, or to defaultNamespaceRoute. - namespaces map[string]string - // mirrorKeys holds, per named upstream, the ns lookup keys of the - // registries it is configured to mirror. - mirrorKeys map[string][]string - - refusalsMu sync.Mutex - // refusals remembers upstream/ns pairs whose refusal was already logged. - refusals map[string]struct{} - // refusalsFull is set once the refusals set reached its limit and that - // was logged. - refusalsFull bool - - // manifestFlight shares one upstream exchange among concurrent - // requests for the same manifest. - manifestFlight singleflight.Group + proxy *Proxy + registryURL string + authURL string + proxyURL string } -type containerRegistry struct { - repositoryPrefix string - registryURL string -} - -// NewContainerHandlerWithRegistry creates a container handler with a custom -// default registry and optional named registries. -func NewContainerHandlerWithRegistry( - proxy *Proxy, - proxyURL, registryURL string, - namedRegistries ...map[string]string, -) *ContainerHandler { - return newContainerHandler(proxy, proxyURL, configuredUpstreamURL(registryURL, dockerHubRegistry), namedRegistries...) -} - -func newContainerHandler( - proxy *Proxy, - proxyURL, registryURL string, - namedRegistries ...map[string]string, -) *ContainerHandler { - h := &ContainerHandler{ +// NewContainerHandler creates a new container registry protocol handler. +func NewContainerHandler(proxy *Proxy, proxyURL string) *ContainerHandler { + return &ContainerHandler{ proxy: proxy, - registryURL: registryURL, + registryURL: dockerHubRegistry, + authURL: dockerHubAuth, proxyURL: strings.TrimSuffix(proxyURL, "/"), } - if len(namedRegistries) > 0 { - h.namedRegistries = make(map[string]string, len(namedRegistries[0])) - for name, registryURL := range namedRegistries[0] { - h.namedRegistries[name] = strings.TrimSuffix(registryURL, "/") - } - } - // The index needs the final default registry URL, so build it last. - h.buildNamespaceIndex() - return h -} - -// buildNamespaceIndex maps the registry hosts containerd may send in the ns -// query parameter to the configured routes. The index is closed-world: a host -// is only ever looked up, never dialed. Docker Hub aliases and the default -// registry's host select the default route; hosts of upstream.oci entries -// select their named upstream. Registry URLs with a path are skipped because -// ns names the registry at the root of that host, not a repository mounted -// below it. On collisions the default route wins, then the alphabetically -// first upstream name. -func (h *ContainerHandler) buildNamespaceIndex() { - h.namespaces = make(map[string]string) - for _, registryURL := range dockerHubNamespaces { - h.indexNamespace(defaultNamespaceRoute, registryURL) - } - if !h.indexNamespace(defaultNamespaceRoute, h.registryURL) { - h.warn("host of the default OCI registry is not indexed for ns lookups: URL has a path; Docker Hub aliases still select it", - "url", redactedURL(h.registryURL)) - } - for _, name := range slices.Sorted(maps.Keys(h.namedRegistries)) { - if !h.indexNamespace(name, h.namedRegistries[name]) { - h.warn("OCI upstream is not reachable through the ns query parameter: URL has a path", - "upstream", name, "url", redactedURL(h.namedRegistries[name])) - } - } -} - -// SetMirroredRegistries records which registry hosts each named upstream -// mirrors, for example "ghcr" -> ["ghcr.io"] for an Artifactory remote of -// ghcr.io. Those hosts select the upstream in ns lookups and are accepted as -// ns on its upstream/{name}/ prefix. They rank below every host taken from a -// configured URL, so they are indexed after the constructor's index. -func (h *ContainerHandler) SetMirroredRegistries(mirrors map[string][]string) { - h.mirrorKeys = make(map[string][]string, len(mirrors)) - for _, name := range slices.Sorted(maps.Keys(mirrors)) { - if _, ok := h.namedRegistries[name]; !ok { - h.warn("OCI mirror entry has no upstream of that name", "upstream", name) - continue - } - for _, host := range mirrors[name] { - registryURL := "https://" + host - keys, ok := namespaceKeysForURL(registryURL) - if !ok { - h.warn("OCI mirror host is not a registry host", "upstream", name, "host", host) - continue - } - h.mirrorKeys[name] = append(h.mirrorKeys[name], keys...) - h.indexNamespace(name, registryURL) - } - } -} - -// indexNamespace maps the ns lookup keys of registryURL to route. Keys that -// another route already owns stay with that route and are reported once. It -// reports false when the URL is not a bare registry root. -func (h *ContainerHandler) indexNamespace(route, registryURL string) bool { - keys, ok := namespaceKeysForURL(registryURL) - if !ok { - return false - } - var taken []string - for _, key := range keys { - owner, exists := h.namespaces[key] - switch { - case !exists: - h.namespaces[key] = route - case owner != route: - if owner == defaultNamespaceRoute { - owner = "default registry" - } - taken = append(taken, key+"="+owner) - } - } - if len(taken) > 0 { - h.warn("OCI upstream shares a registry host with another route; unprefixed ns requests for it go to the other route", - "upstream", route, "hosts", strings.Join(taken, ", ")) - } - return true -} - -// namespaceKeysForURL returns the ns lookup keys of a registry URL. The -// scheme-default port is optional in image references, so such a URL is -// indexed both without and with the port. Any other port is kept as is, which -// keeps https://host:80 and https://host apart. It reports false for URLs -// that are not a bare registry root; namespaceKeysForHost serves those. -func namespaceKeysForURL(registryURL string) ([]string, bool) { - parsed, err := url.Parse(registryURL) - if err != nil || parsed.Host == "" || (parsed.Path != "" && parsed.Path != "/") { - return nil, false - } - return namespaceKeysForHost(parsed), true -} - -// namespaceKeysForHost returns the lookup keys of a URL's host, ignoring its -// path. -func namespaceKeysForHost(parsed *url.URL) []string { - host, port := splitNamespaceHost(parsed.Host) - defaultPort := schemeDefaultPorts[strings.ToLower(parsed.Scheme)] - switch { - case port != "" && port != defaultPort: - return []string{namespaceKey(host, port)} - case defaultPort == "": - return []string{namespaceKey(host, "")} - default: - return []string{namespaceKey(host, ""), namespaceKey(host, defaultPort)} - } -} - -// namespaceKeyForRequest normalizes the ns value of a request. containerd -// sends the registry host of the image reference, with or without a port, so -// the value is matched as sent apart from case and IPv6 bracket form. -func namespaceKeyForRequest(namespace string) string { - host, port := splitNamespaceHost(namespace) - return namespaceKey(host, port) -} - -// splitNamespaceHost splits host[:port], accepting bracketed and bare IPv6 -// hosts without a port. -func splitNamespaceHost(hostport string) (host, port string) { - host, port, err := net.SplitHostPort(hostport) - if err != nil { - return strings.TrimSuffix(strings.TrimPrefix(hostport, "["), "]"), "" - } - return host, port -} - -// namespaceKey builds a lookup key from a host and an optional port: the host -// lowercased, IPv6 literals in brackets. -func namespaceKey(host, port string) string { - host = strings.ToLower(host) - if strings.Contains(host, ":") { - host = "[" + host + "]" - } - if port == "" { - return host - } - return host + ":" + port -} - -// redactedURL returns a registry URL for logging with any password masked. -func redactedURL(raw string) string { - parsed, err := url.Parse(raw) - if err != nil { - return "" - } - return parsed.Redacted() -} - -func (h *ContainerHandler) warn(msg string, args ...any) { - if h.proxy != nil && h.proxy.Logger != nil { - h.proxy.Logger.Warn(msg, args...) - } -} - -// RegisterRegistry routes a repository and its descendants to a specific OCI -// registry. The longest matching repository prefix wins. -func (h *ContainerHandler) RegisterRegistry(repositoryPrefix, registryURL string) { - h.registries = append(h.registries, containerRegistry{ - repositoryPrefix: strings.Trim(repositoryPrefix, "/"), - registryURL: strings.TrimSuffix(registryURL, "/"), - }) -} - -// BlockRegistry prevents a repository and its descendants from falling back to -// the default OCI registry. A more specific registered repository still wins. -func (h *ContainerHandler) BlockRegistry(repositoryPrefix string) { - h.RegisterRegistry(repositoryPrefix, "") -} - -func (h *ContainerHandler) registryURLFor(name string) string { - registryURL := h.registryURL - matchLength := 0 - for _, registry := range h.registries { - if name != registry.repositoryPrefix && !strings.HasPrefix(name, registry.repositoryPrefix+"/") { - continue - } - if len(registry.repositoryPrefix) > matchLength { - registryURL = registry.registryURL - matchLength = len(registry.repositoryPrefix) - } - } - return registryURL } // Routes returns the HTTP handler for container registry requests. @@ -353,18 +51,15 @@ func (h *ContainerHandler) Routes() http.Handler { case path == "" || path == "/": // Version check: GET /v2/ h.handleVersionCheck(w, r) - case strings.HasSuffix(path, "/blobs/"+r.URL.Query().Get("digest")) || strings.Contains(path, "/blobs/sha256:") || strings.Contains(path, "/blobs/sha512:"): + case strings.HasSuffix(path, "/blobs/"+r.URL.Query().Get("digest")) || strings.Contains(path, "/blobs/sha256:"): // Blob download: GET /v2/{name}/blobs/{digest} h.handleBlobDownload(w, r, path) - case manifestPathPattern.MatchString(path): + case strings.Contains(path, "/manifests/"): // Manifest: GET /v2/{name}/manifests/{reference} h.handleManifest(w, r, path) - case tagsListPathPattern.MatchString(path): + case strings.Contains(path, "/tags/list"): // Tags list: GET /v2/{name}/tags/list h.handleTagsList(w, r, path) - case referrersPathPattern.MatchString(path): - // Referrers: GET /v2/{name}/referrers/{digest} - h.handleReferrers(w, r, path) default: http.Error(w, "not found", http.StatusNotFound) } @@ -392,81 +87,48 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req return } - registryURL, upstreamName, cacheName, ok := h.registryForRequest(r, name) - if !ok { - h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry") - return - } + h.proxy.Logger.Info("container blob request", "name", name, "digest", digest) - h.proxy.Logger.Debug("container blob request", "name", upstreamName, "digest", digest) - - filename := digest - cached, err := h.proxy.GetCachedArtifact(r.Context(), "oci", cacheName, digest, filename) - if errors.Is(err, ErrVersionDenied) { - h.containerError(w, http.StatusForbidden, "DENIED", err.Error()) - return - } + // Get auth token for upstream + token, err := h.getAuthToken(r.Context(), name, "pull") if err != nil { - h.proxy.Logger.Error("failed to check blob cache", "error", err) - h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to check blob cache") - return - } - if cached != nil { - w.Header().Set("Docker-Content-Digest", digest) - if cached.Artifact.MediaType == "" { - cached.Artifact.MediaType = "application/octet-stream" - } - ServeArtifactRequest(w, r, cached) + h.proxy.Logger.Error("failed to get auth token", "error", err) + h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate") return } // For HEAD requests, just proxy to upstream if r.Method == http.MethodHead { - h.proxyBlobHead(w, r, registryURL, upstreamName, digest) + h.proxyBlobHead(w, r, name, digest, token) return } - // Try to get from cache, or fetch from the authentication-aware upstream client. - result, err := h.proxy.GetOrFetchArtifactFromURLWithDigest( + // Try to get from cache, or fetch from upstream with auth + filename := digest + headers := http.Header{"Authorization": {"Bearer " + token}} + result, err := h.proxy.GetOrFetchArtifactFromURLWithHeaders( r.Context(), "oci", - cacheName, + name, digest, // use digest as version filename, - fmt.Sprintf("%s/v2/%s/blobs/%s", registryURL, upstreamName, digest), - digest, + fmt.Sprintf("%s/v2/%s/blobs/%s", h.registryURL, name, digest), + headers, ) if err != nil { - if errors.Is(err, ErrUpstreamNotFound) { - h.containerError(w, http.StatusNotFound, "BLOB_UNKNOWN", "blob unknown to registry") - return - } - if errors.Is(err, ErrArtifactBlocked) || errors.Is(err, ErrVersionDenied) { - h.containerError(w, http.StatusForbidden, "DENIED", err.Error()) - return - } - if errors.Is(err, ErrArtifactDigestMismatch) { - h.proxy.Logger.Error("upstream blob failed digest verification", "error", err) - h.containerError(w, http.StatusBadGateway, "DIGEST_INVALID", "blob digest verification failed") - return - } - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("failed to fetch blob", "error", err) - h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch blob") + h.containerError(w, http.StatusBadGateway, "BLOB_UNKNOWN", "failed to fetch blob") return } w.Header().Set("Docker-Content-Digest", digest) - if result.Artifact.MediaType == "" { - result.Artifact.MediaType = "application/octet-stream" - } - ServeArtifactRequest(w, r, result) + w.Header().Set("Content-Type", "application/octet-stream") + ServeArtifact(w, result) } -// handleManifest serves immutable manifests from cache and revalidates mutable tags. +// handleManifest proxies manifest requests to upstream. +// Manifests change when tags are updated, so we proxy these directly. // Path format: {name}/manifests/{reference} func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request, path string) { if r.Method != http.MethodGet && r.Method != http.MethodHead { @@ -480,20 +142,61 @@ func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request return } - registryURL, upstreamName, cacheName, ok := h.registryForRequest(r, name) - if !ok { - h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry") + h.proxy.Logger.Info("container manifest request", "name", name, "reference", reference) + + // Get auth token + token, err := h.getAuthToken(r.Context(), name, "pull") + if err != nil { + h.proxy.Logger.Error("failed to get auth token", "error", err) + h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate") return } - h.proxy.Logger.Debug("container manifest request", "name", upstreamName, "reference", reference) - if h.manifestDenied(w, cacheName, reference) { + // Proxy to upstream + upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", h.registryURL, name, reference) + + req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) + if err != nil { + h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request") return } - h.serveManifest(w, r, registryURL, upstreamName, cacheName, reference) + + req.Header.Set("Authorization", "Bearer "+token) + + // Forward Accept header for content negotiation + if accept := r.Header.Get("Accept"); accept != "" { + req.Header.Set("Accept", accept) + } else { + // Default accept headers for manifests + req.Header.Set("Accept", strings.Join([]string{ + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.docker.distribution.manifest.v1+prettyjws", + }, ", ")) + } + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + h.proxy.Logger.Error("failed to fetch manifest", "error", err) + h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") + return + } + defer func() { _ = resp.Body.Close() }() + + // Copy relevant headers + for _, header := range []string{"Content-Type", "Content-Length", "Docker-Content-Digest", "ETag"} { + if v := resp.Header.Get(header); v != "" { + w.Header().Set(header, v) + } + } + + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) } -// handleTagsList caches tag list responses for offline OCI pulls. +// handleTagsList proxies tag list requests to upstream. func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request, path string) { if r.Method != http.MethodGet { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) @@ -506,18 +209,78 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request return } - registryURL, upstreamName, _, ok := h.registryForRequest(r, name) - if !ok { - h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry") + // Get auth token + token, err := h.getAuthToken(r.Context(), name, "pull") + if err != nil { + h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate") return } - h.serveTagsList(w, r, registryURL, upstreamName) + upstreamURL := fmt.Sprintf("%s/v2/%s/tags/list", h.registryURL, name) + if r.URL.RawQuery != "" { + upstreamURL += "?" + r.URL.RawQuery + } + + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) + if err != nil { + h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request") + return + } + + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") + return + } + defer func() { _ = resp.Body.Close() }() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) +} + +// getAuthToken gets a bearer token for the specified repository. +// Docker Hub requires auth even for public images. +func (h *ContainerHandler) getAuthToken(_ interface{ Done() <-chan struct{} }, repository, action string) (string, error) { + // For Docker Hub: https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repo}:pull + authURL := fmt.Sprintf("%s/token?service=registry.docker.io&scope=repository:%s:%s", + h.authURL, repository, action) + + req, err := http.NewRequest(http.MethodGet, authURL, nil) + if err != nil { + return "", err + } + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("auth failed with status %d", resp.StatusCode) + } + + var tokenResp struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + } + + if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { + return "", err + } + + if tokenResp.Token != "" { + return tokenResp.Token, nil + } + return tokenResp.AccessToken, nil } // proxyBlobHead handles HEAD requests for blobs. -func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, registryURL, name, digest string) { - upstreamURL := fmt.Sprintf("%s/v2/%s/blobs/%s", registryURL, name, digest) +func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, name, digest, token string) { + upstreamURL := fmt.Sprintf("%s/v2/%s/blobs/%s", h.registryURL, name, digest) req, err := http.NewRequestWithContext(r.Context(), http.MethodHead, upstreamURL, nil) if err != nil { @@ -525,164 +288,27 @@ func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, return } + req.Header.Set("Authorization", "Bearer "+token) + resp, err := h.proxy.HTTPClient.Do(req) if err != nil { - if writeUpstreamHeld(w, err) { - return - } h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") return } defer func() { _ = resp.Body.Close() }() - h.proxy.relayResponse(w, r, resp, func(dst, src http.Header) { - for _, header := range []string{headerContentType, headerContentLength, "Docker-Content-Digest", headerETag, headerLastModified} { - if v := src.Get(header); v != "" { - dst.Set(header, v) - } + for _, header := range []string{"Content-Type", "Content-Length", "Docker-Content-Digest"} { + if v := resp.Header.Get(header); v != "" { + w.Header().Set(header, v) } - if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices && dst.Get("Docker-Content-Digest") == "" { - dst.Set("Docker-Content-Digest", digest) - } - }) -} + } -// registryForRequest resolves the repository name of a request. containerd -// mirror requests name the target registry in the ns query parameter; without -// it the name is routed by registryForName. -func (h *ContainerHandler) registryForRequest(r *http.Request, name string) (registryURL, upstreamName, cacheName string, ok bool) { - namespaces := r.URL.Query()[namespaceQueryParam] - switch { - case len(namespaces) == 0 || (len(namespaces) == 1 && namespaces[0] == ""): - return h.registryForName(name) - case len(namespaces) > 1: - return "", "", "", false - } - return h.registryForNamespace(namespaces[0], name) -} - -// registryForNamespace resolves a repository name verbatim against the registry -// named by ns. Cache names match the unprefixed and upstream/{name}/ routes, -// so all routes to one registry share blobs. -// -// Per-registry containerd mirrors with override_path address the reserved -// upstream/{name}/ prefix and still send ns. Such requests are served like -// the prefix route without ns unless ns contradicts the prefix, see -// namespaceNamesPrefixUpstream. -func (h *ContainerHandler) registryForNamespace(namespace, name string) (registryURL, upstreamName, cacheName string, ok bool) { - if len(namespace) > maxNamespaceLength { - return "", "", "", false - } - if strings.HasPrefix(name, "upstream/") { - if !h.namespaceNamesPrefixUpstream(namespace, name) { - return "", "", "", false - } - return h.registryForName(name) - } - route, ok := h.namespaces[namespaceKeyForRequest(namespace)] - if !ok { - return "", "", "", false - } - if route == defaultNamespaceRoute { - // ns explicitly names the default registry, so repository prefix - // routes such as Homebrew's do not apply. - if h.registryURL == "" { - return "", "", "", false - } - return h.registryURL, name, name, true - } - registryURL = h.namedRegistries[route] - if registryURL == "" { - return "", "", "", false - } - return registryURL, name, "upstream/" + route + "/" + name, true -} - -// namespaceNamesPrefixUpstream decides whether an upstream/{name}/ request -// may carry the given ns. A per-registry mirror with override_path and a -// _default wildcard pull of /upstream/{name}/... send the same request, -// so ns has to name a registry the upstream is known to serve: its own host -// (the URL's path is irrelevant), a host listed for it in upstream.oci_mirrors, -// or Docker Hub. Docker Hub repository names have two path components, so -// docker.io/upstream/... is never a real image and a Docker Hub mirror behind -// any prefix stays reachable. Any other ns gets a 404 and containerd falls -// back to the registry the image actually names. -func (h *ContainerHandler) namespaceNamesPrefixUpstream(namespace, name string) bool { - rest, _ := strings.CutPrefix(name, "upstream/") - upstream, _, _ := strings.Cut(rest, "/") - parsed, err := url.Parse(h.namedRegistries[upstream]) - if err != nil || parsed.Host == "" { - return false - } - key := namespaceKeyForRequest(namespace) - if isDockerHubKey(key) || - slices.Contains(namespaceKeysForHost(parsed), key) || - slices.Contains(h.mirrorKeys[upstream], key) { - return true - } - h.reportNamespaceRefusal(upstream, key, namespace) - return false -} - -// reportNamespaceRefusal logs a refused ns on a prefix route. containerd -// falls back to the registry itself on that 404 without reporting anything, -// so this line is all an operator gets to see. Each upstream/ns pair is -// logged once, since a misconfigured node repeats the same request for every -// layer. ns comes from the client, so the set is bounded: once it is full, -// that is logged one time and later refusals stay quiet. Overlong values -// never get here (see maxNamespaceLength). -func (h *ContainerHandler) reportNamespaceRefusal(upstream, key, namespace string) { - pair := upstream + "\x00" + key - h.refusalsMu.Lock() - if _, seen := h.refusals[pair]; seen { - h.refusalsMu.Unlock() - return - } - if len(h.refusals) >= maxReportedNamespaceRefusals { - announce := !h.refusalsFull - h.refusalsFull = true - h.refusalsMu.Unlock() - if announce { - h.warn("too many different ns values refused on OCI upstream prefixes; further refusals are not logged", - "limit", maxReportedNamespaceRefusals) - } - return - } - if h.refusals == nil { - h.refusals = make(map[string]struct{}) - } - h.refusals[pair] = struct{}{} - h.refusalsMu.Unlock() - h.warn("refusing ns on OCI upstream prefix; if the upstream mirrors this registry, list it in upstream.oci_mirrors", - "upstream", upstream, "ns", namespace) -} - -// registryForName resolves a client-visible OCI repository name to an upstream -// registry and its repository name. Named upstreams use upstream/{name}/ as a -// reserved prefix. Other names are matched against registered repository -// prefixes, falling back to Docker Hub when no prefix matches. -func (h *ContainerHandler) registryForName(name string) (registryURL, upstreamName, cacheName string, ok bool) { - parts := strings.SplitN(name, "/", registrySelectorParts) - if len(parts) >= 2 && parts[0] == "upstream" { - if len(parts) != registrySelectorParts || parts[2] == "" { - return "", "", "", false - } - registryURL, ok = h.namedRegistries[parts[1]] - if !ok || registryURL == "" { - return "", "", "", false - } - return registryURL, parts[2], name, true - } - registryURL = h.registryURLFor(name) - if registryURL == "" { - return "", "", "", false - } - return registryURL, name, name, true + w.WriteHeader(resp.StatusCode) } // containerError writes an OCI-compliant error response. func (h *ContainerHandler) containerError(w http.ResponseWriter, status int, code, message string) { - w.Header().Set(headerContentType, "application/json") + w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) _ = json.NewEncoder(w).Encode(map[string]any{ "errors": []map[string]string{ @@ -692,7 +318,7 @@ func (h *ContainerHandler) containerError(w http.ResponseWriter, status int, cod } // blobPathPattern matches blob paths: {name}/blobs/{digest} -var blobPathPattern = regexp.MustCompile(`^(.+)/blobs/(sha256:[a-f0-9]{64}|sha512:[a-f0-9]{128})$`) +var blobPathPattern = regexp.MustCompile(`^(.+)/blobs/(sha256:[a-f0-9]+)$`) // parseBlobPath extracts repository name and digest from a blob path. func (h *ContainerHandler) parseBlobPath(path string) (name, digest string) { @@ -704,7 +330,7 @@ func (h *ContainerHandler) parseBlobPath(path string) (name, digest string) { } // manifestPathPattern matches manifest paths: {name}/manifests/{reference} -var manifestPathPattern = regexp.MustCompile(`^(.+)/manifests/([^/]+)$`) +var manifestPathPattern = regexp.MustCompile(`^(.+)/manifests/(.+)$`) // parseManifestPath extracts repository name and reference from a manifest path. func (h *ContainerHandler) parseManifestPath(path string) (name, reference string) { diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go deleted file mode 100644 index 6d129d0..0000000 --- a/internal/handler/container_manifest.go +++ /dev/null @@ -1,469 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "mime" - "net/http" - "regexp" - "slices" - "sort" - "strconv" - "strings" - "time" -) - -const ( - containerManifestCacheEcosystem = "oci-manifest" - containerStaleWarning = `110 - "Response is Stale"` - - containerAcceptWildcardSpecificity = iota - containerAcceptTypeWildcardSpecificity - containerAcceptExactSpecificity -) - -const ( - // containerManifestIndexCacheClass replaces the Accept header in tag cache - // keys for clients that accept an index type. - containerManifestIndexCacheClass = "index" - ociImageIndexMediaType = "application/vnd.oci.image.index.v1+json" - dockerManifestListMediaType = "application/vnd.docker.distribution.manifest.list.v2+json" -) - -var manifestDigestReferencePattern = regexp.MustCompile(`^[a-z0-9]+:[a-f0-9]+$`) - -// referrersTagPattern matches the sha256- tags of the OCI referrers tag -// schema. Docker looks one up on every pull and fails the pull on any answer -// other than a manifest or 404. -var referrersTagPattern = regexp.MustCompile(`^sha256-[a-f0-9]{64}$`) - -type cachedContainerManifest struct { - body []byte - contentType string - contentDigest string - etag string - size int64 - lastModified time.Time - fetchedAt time.Time -} - -// serveManifest serves a manifest unless the denylist names the digest it -// resolves to under cacheName, the name blobs are stored under. -func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, registryURL, name, cacheName, reference string) { - accept := containerManifestAccept(r) - req := manifestRequest{ - registryURL: registryURL, - name: name, - reference: reference, - accept: accept, - cacheAccept: normalizeContainerManifestAccept(accept), - method: r.Method, - } - cached, _ := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, req.accept, req.cacheAccept) - if cached != nil && (req.immutable() || h.containerManifestFresh(cached)) { - h.writeManifest(w, r, cacheName, cached, false) - return - } - - outcome := h.coalescedManifest(r.Context(), req) - switch { - case outcome.manifest != nil: - h.writeManifest(w, r, cacheName, outcome.manifest, false) - case outcome.relay != nil: - outcome.relay.write(h.proxy, w, r) - case outcome.digestInvalid: - h.containerError(w, http.StatusBadGateway, "DIGEST_INVALID", "manifest digest verification failed") - default: - h.serveStaleManifestOrError(w, r, cacheName, reference, cached, outcome.err) - } -} - -func (h *ContainerHandler) serveStaleManifestOrError(w http.ResponseWriter, r *http.Request, cacheName, reference string, cached *cachedContainerManifest, err error) { - if cached != nil { - h.proxy.Logger.Warn("upstream manifest fetch failed, serving stale cache", "error", err) - h.writeManifest(w, r, cacheName, cached, true) - return - } - if referrersTagPattern.MatchString(reference) { - h.proxy.Logger.Warn("upstream manifest fetch failed, answering referrers tag as unknown", "reference", reference, "error", err) - h.containerError(w, http.StatusNotFound, "MANIFEST_UNKNOWN", "manifest unknown") - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("failed to fetch manifest", "error", err) - h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") -} - -// manifestDenied writes a 403 and reports true when the denylist names -// reference, a tag or digest, under the repository's cache name. -func (h *ContainerHandler) manifestDenied(w http.ResponseWriter, cacheName, reference string) bool { - if reference == "" || !h.proxy.versionDenied("oci", cacheName, reference) { - return false - } - h.containerError(w, http.StatusForbidden, "DENIED", - ErrVersionDenied.Error()+": "+canonicalVersionPURL("oci", cacheName, reference)) - return true -} - -// writeManifest writes manifest unless the denylist names its digest. -func (h *ContainerHandler) writeManifest(w http.ResponseWriter, r *http.Request, cacheName string, manifest *cachedContainerManifest, stale bool) { - if h.manifestDenied(w, cacheName, manifest.contentDigest) { - return - } - writeContainerManifest(w, r, manifest, stale) -} - -func (h *ContainerHandler) containerManifestFresh(manifest *cachedContainerManifest) bool { - return h.proxy.MetadataTTL > 0 && !manifest.fetchedAt.IsZero() && time.Since(manifest.fetchedAt) < h.proxy.MetadataTTL -} - -func (h *ContainerHandler) containerManifestCacheKey(registryURL, name, reference, accept string) string { - identity := strings.Join([]string{registryURL, name, reference, accept}, "\x00") - sum := sha256.Sum256([]byte(identity)) - return hex.EncodeToString(sum[:]) -} - -// containerManifestSharedCacheKey is the key clients with different Accept -// headers share. A digest names exactly one document. A tag is stored once for -// all clients that accept an index type, whatever upstream returned for it, so -// Docker, containerd and Podman read the same entry. Clients without index -// support keep a key per normalized Accept header, because registries may -// convert an index into a single platform manifest for them. -func (h *ContainerHandler) containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept string) string { - if manifestDigestReferencePattern.MatchString(reference) { - identity := strings.Join([]string{registryURL, name, reference}, "\x00") - sum := sha256.Sum256([]byte(identity)) - return hex.EncodeToString(sum[:]) - } - if containerManifestAcceptsIndex(cacheAccept) { - return h.containerManifestCacheKey(registryURL, name, reference, containerManifestIndexCacheClass) - } - return h.containerManifestCacheKey(registryURL, name, reference, cacheAccept) -} - -func containerManifestAcceptsIndex(accept string) bool { - return containerManifestAccepts(accept, ociImageIndexMediaType) || containerManifestAccepts(accept, dockerManifestListMediaType) -} - -// containerManifestLegacyCacheKeys are the per-Accept keys older versions -// used. They are still written so a rollback finds the cache, and read when -// the shared key has no entry yet. -func (h *ContainerHandler) containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, sharedKey string) []string { - var keys []string - for _, variant := range []string{cacheAccept, accept} { - key := h.containerManifestCacheKey(registryURL, name, reference, variant) - if key != sharedKey && !slices.Contains(keys, key) { - keys = append(keys, key) - } - } - return keys -} - -// loadContainerManifestForAccept returns the cached manifest for the client and -// whether it came from a per-Accept fallback key while the shared key holds a -// representation the client does not accept. -func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheAccept string) (*cachedContainerManifest, bool) { - cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept) - cached, err := h.loadContainerManifest(ctx, cacheKey) - if err != nil { - h.proxy.Logger.Warn("failed to read cached container manifest", "error", err) - return nil, false - } - if cached != nil && containerManifestCacheCompatible(accept, cached) { - return cached, false - } - // The shared entry may hold a representation this client does not accept, - // for example an OCI index for a client that only accepts Docker manifest - // lists. Its own per-Accept entry is still written, so look there. - sharedEntryExists := cached != nil - - for _, legacyCacheKey := range h.containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, cacheKey) { - cached, err = h.loadContainerManifest(ctx, legacyCacheKey) - if err != nil { - h.proxy.Logger.Warn("failed to read legacy cached container manifest", "error", err) - return nil, false - } - if cached == nil || !containerManifestCacheCompatible(accept, cached) { - continue - } - if !sharedEntryExists { - if err := h.storeContainerManifest(ctx, cacheKey, cached); err != nil { - h.proxy.Logger.Warn("failed to migrate cached container manifest", "error", err) - } - } - return cached, sharedEntryExists - } - return nil, false -} - -func (h *ContainerHandler) storeContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheAccept string, manifest *cachedContainerManifest) { - cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept) - if err := h.storeContainerManifest(ctx, cacheKey, manifest); err != nil { - h.proxy.Logger.Warn("failed to cache container manifest", "error", err) - } - - h.storeContainerManifestLegacyEntries(ctx, registryURL, name, reference, accept, cacheAccept, manifest) -} - -// storeContainerManifestLegacyEntries writes only the per-Accept keys. -func (h *ContainerHandler) storeContainerManifestLegacyEntries(ctx context.Context, registryURL, name, reference, accept, cacheAccept string, manifest *cachedContainerManifest) { - cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept) - for _, legacyCacheKey := range h.containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, cacheKey) { - if err := h.storeContainerManifest(ctx, legacyCacheKey, manifest); err != nil { - h.proxy.Logger.Warn("failed to cache legacy container manifest", "error", err) - } - } -} - -func (h *ContainerHandler) loadContainerManifest(ctx context.Context, cacheKey string) (*cachedContainerManifest, error) { - if h.proxy.DB == nil || h.proxy.Storage == nil { - return nil, nil - } - entry, err := h.proxy.DB.GetMetadataCache(containerManifestCacheEcosystem, cacheKey) - if err != nil || entry == nil { - return nil, err - } - reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) - if err != nil { - return nil, nil - } - defer func() { _ = reader.Close() }() - body, err := h.proxy.ReadMetadata(reader) - if err != nil { - return nil, err - } - h.proxy.recordMetadataRead(entry) - - manifest := &cachedContainerManifest{body: body, size: int64(len(body))} - if entry.ContentType.Valid { - manifest.contentType = entry.ContentType.String - } - if entry.ContentDigest.Valid { - manifest.contentDigest = entry.ContentDigest.String - } else { - manifest.contentDigest = sha256Digest(body) - } - if entry.ETag.Valid { - manifest.etag = entry.ETag.String - } - if entry.Size.Valid { - manifest.size = entry.Size.Int64 - } - if entry.LastModified.Valid { - manifest.lastModified = entry.LastModified.Time - } - if entry.FetchedAt.Valid { - manifest.fetchedAt = entry.FetchedAt.Time - } - return manifest, nil -} - -func (h *ContainerHandler) storeContainerManifest(ctx context.Context, cacheKey string, manifest *cachedContainerManifest) error { - size, err := h.storeContainerMetadata(ctx, containerManifestCacheEcosystem, cacheKey, manifest.body, - manifest.etag, "", manifest.contentType, manifest.contentDigest, manifest.lastModified, manifest.fetchedAt) - if err != nil { - return fmt.Errorf("storing manifest: %w", err) - } - manifest.size = size - return nil -} - -func writeContainerManifest(w http.ResponseWriter, r *http.Request, manifest *cachedContainerManifest, stale bool) { - if manifest.contentType != "" { - w.Header().Set(headerContentType, manifest.contentType) - } - // A manifest known only from an upstream HEAD may lack a size. - if manifest.size >= 0 { - w.Header().Set(headerContentLength, strconv.FormatInt(manifest.size, 10)) - } - if manifest.contentDigest != "" { - w.Header().Set("Docker-Content-Digest", manifest.contentDigest) - } - if manifest.etag != "" { - w.Header().Set(headerETag, manifest.etag) - } - if !manifest.lastModified.IsZero() { - w.Header().Set(headerLastModified, manifest.lastModified.UTC().Format(http.TimeFormat)) - } - if stale { - w.Header().Set("Warning", containerStaleWarning) - } - if ifNoneMatchHits(r.Header.Get("If-None-Match"), manifest.etag) { - w.WriteHeader(http.StatusNotModified) - return - } - if !manifest.lastModified.IsZero() { - if modifiedSince, err := http.ParseTime(r.Header.Get("If-Modified-Since")); err == nil && !manifest.lastModified.After(modifiedSince) { - w.WriteHeader(http.StatusNotModified) - return - } - } - w.WriteHeader(http.StatusOK) - if r.Method != http.MethodHead { - _, _ = w.Write(manifest.body) - } -} - -// containerManifestAccept joins every Accept header: Docker sends one per -// media type, and the first alone would rule out the others. -func containerManifestAccept(r *http.Request) string { - if accept := strings.Join(r.Header.Values("Accept"), ", "); accept != "" { - return accept - } - return strings.Join([]string{ - "application/vnd.oci.image.manifest.v1+json", - "application/vnd.oci.image.index.v1+json", - "application/vnd.docker.distribution.manifest.v2+json", - "application/vnd.docker.distribution.manifest.list.v2+json", - "application/vnd.docker.distribution.manifest.v1+prettyjws", - }, ", ") -} - -func normalizeContainerManifestAccept(accept string) string { - mediaTypes := make(map[string]struct{}) - for _, value := range strings.Split(accept, ",") { - value = strings.TrimSpace(value) - if value == "" { - continue - } - mediaType, params, err := mime.ParseMediaType(value) - if err != nil { - mediaTypes[strings.ToLower(value)] = struct{}{} - continue - } - paramKeys := make([]string, 0, len(params)) - for key := range params { - paramKeys = append(paramKeys, key) - } - sort.Strings(paramKeys) - canonical := strings.ToLower(mediaType) - for _, key := range paramKeys { - value := params[key] - if strings.EqualFold(key, "q") { - if quality, err := strconv.ParseFloat(value, 64); err == nil { - if quality == 1 { - continue - } - value = strconv.FormatFloat(quality, 'g', -1, 64) - } - } - canonical += ";" + strings.ToLower(key) + "=" + value - } - mediaTypes[canonical] = struct{}{} - } - canonicalMediaTypes := make([]string, 0, len(mediaTypes)) - for mediaType := range mediaTypes { - canonicalMediaTypes = append(canonicalMediaTypes, mediaType) - } - sort.Strings(canonicalMediaTypes) - return strings.Join(canonicalMediaTypes, ",") -} - -func containerManifestAccepts(accept, contentType string) bool { - contentType, contentParams, err := mime.ParseMediaType(contentType) - if err != nil { - return false - } - contentType = strings.ToLower(contentType) - contentMajor, contentMinor, found := strings.Cut(contentType, "/") - if !found { - return false - } - - bestMediaTypeSpecificity := -1 - bestParameterSpecificity := 0 - bestQuality := 0.0 - for _, value := range strings.Split(accept, ",") { - mediaType, params, err := mime.ParseMediaType(strings.TrimSpace(value)) - if err != nil { - continue - } - mediaType = strings.ToLower(mediaType) - major, minor, found := strings.Cut(mediaType, "/") - if found && containerAcceptRangeMatches(major, minor, params, contentMajor, contentMinor, contentParams) { - mediaTypeSpecificity, parameterSpecificity := containerAcceptSpecificity(major, minor, params) - if mediaTypeSpecificity > bestMediaTypeSpecificity || - (mediaTypeSpecificity == bestMediaTypeSpecificity && parameterSpecificity > bestParameterSpecificity) { - bestMediaTypeSpecificity = mediaTypeSpecificity - bestParameterSpecificity = parameterSpecificity - bestQuality = containerAcceptQuality(params) - } - } - } - return bestQuality > 0 -} - -func containerManifestCacheCompatible(accept string, manifest *cachedContainerManifest) bool { - return manifest.contentType == "" || containerManifestAccepts(accept, manifest.contentType) -} - -func containerAcceptRangeMatches(major, minor string, params map[string]string, contentMajor, contentMinor string, contentParams map[string]string) bool { - if (major != "*" && major != contentMajor) || (minor != "*" && minor != contentMinor) { - return false - } - for key, value := range params { - if strings.EqualFold(key, "q") { - continue - } - if contentParams[key] != value { - return false - } - } - return true -} - -func containerAcceptSpecificity(major, minor string, params map[string]string) (int, int) { - parameterSpecificity := 0 - for key := range params { - if !strings.EqualFold(key, "q") { - parameterSpecificity++ - } - } - - switch { - case major == "*" && minor == "*": - return containerAcceptWildcardSpecificity, parameterSpecificity - case major == "*" || minor == "*": - return containerAcceptTypeWildcardSpecificity, parameterSpecificity - default: - return containerAcceptExactSpecificity, parameterSpecificity - } -} - -func containerAcceptQuality(params map[string]string) float64 { - value, ok := params["q"] - if !ok { - return 1 - } - quality, err := strconv.ParseFloat(value, 64) - if err != nil || quality < 0 || quality > 1 { - return 0 - } - return quality -} - -func copyContainerManifestHeaders(destination, source http.Header) { - for _, header := range []string{headerContentType, headerContentLength, "Docker-Content-Digest", headerETag, headerLastModified, "WWW-Authenticate"} { - if value := source.Get(header); value != "" { - destination.Set(header, value) - } - } -} - -func parseHTTPTime(value string) time.Time { - parsed, _ := http.ParseTime(value) - return parsed -} - -func shouldServeStaleManifest(status int) bool { - return status == http.StatusTooManyRequests || status >= http.StatusInternalServerError -} - -func sha256Digest(body []byte) string { - digest := sha256.Sum256(body) - return "sha256:" + hex.EncodeToString(digest[:]) -} diff --git a/internal/handler/container_manifest_fetch.go b/internal/handler/container_manifest_fetch.go deleted file mode 100644 index 7568671..0000000 --- a/internal/handler/container_manifest_fetch.go +++ /dev/null @@ -1,366 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "fmt" - "io" - "net/http" - "strconv" - "strings" - "time" - - "github.com/git-pkgs/proxy/internal/metrics" -) - -// manifestRequest is one client manifest request as the upstream sees it. -type manifestRequest struct { - registryURL string - name string - reference string - accept string - cacheAccept string - method string -} - -func (req manifestRequest) immutable() bool { - return manifestDigestReferencePattern.MatchString(req.reference) -} - -// coalesceKey identifies requests that can share one upstream exchange. The -// normalized Accept is part of it because registries answer a tag with an -// index or a single manifest depending on it, and the method because a HEAD -// never fetches a manifest body the client did not ask for. -func (req manifestRequest) coalesceKey() string { - return strings.Join([]string{containerManifestCacheEcosystem, req.method, req.registryURL, req.name, req.reference, req.cacheAccept}, "\x00") -} - -func (req manifestRequest) url(reference string) string { - return fmt.Sprintf("%s/v2/%s/manifests/%s", req.registryURL, req.name, reference) -} - -// manifestOutcome is what callers sharing a manifest request write. Exactly -// one of manifest, relay and digestInvalid is set; otherwise err is the -// upstream failure, answered from the caller's stale copy if it has one. -type manifestOutcome struct { - manifest *cachedContainerManifest - relay *capturedResponse - digestInvalid bool - err error -} - -// capturedResponse is an upstream response other than a manifest, such as a -// 404 or 401, read in full so every caller sharing the request can relay it. -// A body that could not be read to the end is relayed up to that point and -// then fails the same way, so clients still see the truncation. -type capturedResponse struct { - status int - header http.Header - body []byte - readErr error - trailer http.Header -} - -func (h *ContainerHandler) captureResponse(resp *http.Response) *capturedResponse { - captured := &capturedResponse{status: resp.StatusCode, header: resp.Header.Clone()} - if resp.Request != nil && resp.Request.Method == http.MethodHead { - return captured - } - limit := h.proxy.MetadataMaxSize - if limit <= 0 { - limit = defaultMetadataMaxSize - } - captured.body, captured.readErr = io.ReadAll(io.LimitReader(resp.Body, limit+1)) - if captured.readErr == nil && int64(len(captured.body)) > limit { - captured.body, captured.readErr = captured.body[:limit], ErrMetadataTooLarge - } - captured.trailer = resp.Trailer.Clone() - return captured -} - -type failingReader struct{ err error } - -func (f failingReader) Read([]byte) (int, error) { return 0, f.err } - -func (c *capturedResponse) write(p *Proxy, w http.ResponseWriter, r *http.Request) { - var body io.Reader = bytes.NewReader(c.body) - if c.readErr != nil { - body = io.MultiReader(body, failingReader{c.readErr}) - } - resp := &http.Response{ - StatusCode: c.status, - Header: c.header.Clone(), - Trailer: c.trailer.Clone(), - Body: io.NopCloser(body), - } - p.relayResponse(w, r, resp, copyContainerManifestHeaders) -} - -// coalescedManifest runs the upstream part of a manifest request once for -// concurrent callers with the same key. Like coalesceMetadata, it runs on a -// context detached from the first caller, and each waiter still leaves when -// its own context ends. The result is a whole manifest with its headers, -// which a metadataResult does not carry, hence the separate group. -func (h *ContainerHandler) coalescedManifest(ctx context.Context, req manifestRequest) manifestOutcome { - results := h.manifestFlight.DoChan(req.coalesceKey(), func() (any, error) { - return h.fetchManifest(context.WithoutCancel(ctx), req), nil - }) - select { - case <-ctx.Done(): - return manifestOutcome{err: ctx.Err()} - case res := <-results: - outcome, ok := res.Val.(manifestOutcome) - if !ok { - return manifestOutcome{err: errSharedFetchAbandoned} - } - return outcome - } -} - -// fetchManifest answers a manifest request that the cache could not. Other -// instances sharing the database take turns through a fetch lease, so a tag -// that expired everywhere at once is revalidated once. -func (h *ContainerHandler) fetchManifest(ctx context.Context, req manifestRequest) (outcome manifestOutcome) { - usable := func(m *cachedContainerManifest) bool { - return m != nil && (req.immutable() || h.containerManifestFresh(m)) - } - // A shared request that finished since the caller looked has refreshed - // the entry already. - cached, fromFallback := h.loadContainerManifestForAccept(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept) - if usable(cached) { - return manifestOutcome{manifest: cached} - } - - // Without a TTL nothing another instance stores is fresh, so there is - // nothing to wait for. - if req.immutable() || h.proxy.MetadataTTL > 0 { - var stored *cachedContainerManifest - release, lease, err := h.proxy.awaitFetchLease(ctx, metrics.CacheKindMetadata, req.coalesceKey(), func() bool { - stored, _ = h.loadContainerManifestForAccept(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept) - return usable(stored) - }) - if err != nil { - return manifestOutcome{err: err} - } - switch lease { - case leaseStored: - return manifestOutcome{manifest: stored} - case leaseFailed: - if cached != nil { - return manifestOutcome{err: errFetchFailedElsewhere} - } - case leaseFetch: - } - defer func() { release(outcome.err != nil) }() - } - - switch { - case req.method == http.MethodHead: - return h.headManifest(ctx, req, cached, fromFallback) - case req.immutable(): - return h.getManifest(ctx, req, req.reference, cached, fromFallback) - default: - return h.revalidateTag(ctx, req, cached, fromFallback) - } -} - -// revalidateTag answers a GET for a tag that is stale or not cached. Docker -// Hub counts manifest GETs against its pull limit but not HEADs, so a stale -// tag is resolved with a HEAD first and the manifest fetched by digest only -// when it is not cached yet. A registry whose HEAD gives no digest gets the -// GET for the tag, as before. A tag never seen is fetched with one GET: a -// HEAD first would only add a round trip. -func (h *ContainerHandler) revalidateTag(ctx context.Context, req manifestRequest, cached *cachedContainerManifest, fromFallback bool) manifestOutcome { - if cached == nil { - return h.getManifest(ctx, req, req.reference, nil, false) - } - resp, err := h.requestManifest(ctx, http.MethodHead, req, req.reference, "") - if err != nil { - return manifestOutcome{err: err} - } - _ = resp.Body.Close() - - digest := resp.Header.Get("Docker-Content-Digest") - switch { - case resp.StatusCode == http.StatusOK && manifestDigestReferencePattern.MatchString(digest): - if outcome, ok := h.tagFromCachedDigest(ctx, req, cached, fromFallback, resp, digest); ok { - return outcome - } - if outcome := h.getManifest(ctx, req, digest, nil, false); outcome.manifest != nil { - return outcome - } - // The tag may have moved again, or the registry may not serve - // manifests by digest to this client; ask for the tag itself. - case shouldServeStaleManifest(resp.StatusCode): - return manifestOutcome{err: fmt.Errorf("upstream status %d", resp.StatusCode)} - } - return h.getManifest(ctx, req, req.reference, cached, fromFallback) -} - -// headManifest answers a client HEAD. When the digest upstream names is -// cached, the tag entry is pointed at it, so the next GET is a cache hit. -// Otherwise the upstream headers are passed on and nothing is stored. -func (h *ContainerHandler) headManifest(ctx context.Context, req manifestRequest, cached *cachedContainerManifest, fromFallback bool) manifestOutcome { - resp, err := h.requestManifest(ctx, http.MethodHead, req, req.reference, "") - if err != nil { - return manifestOutcome{err: err} - } - _ = resp.Body.Close() - - digest := resp.Header.Get("Docker-Content-Digest") - switch { - case resp.StatusCode == http.StatusOK && manifestDigestReferencePattern.MatchString(digest): - if req.immutable() && digestMismatch(req.reference, digest) { - h.proxy.Logger.Error("upstream manifest HEAD names another digest", - "name", req.name, "reference", req.reference, "upstream", digest) - return manifestOutcome{digestInvalid: true} - } - if !req.immutable() { - if outcome, ok := h.tagFromCachedDigest(ctx, req, cached, fromFallback, resp, digest); ok { - return outcome - } - } - size := int64(-1) - if value := resp.Header.Get(headerContentLength); value != "" { - if parsed, err := strconv.ParseInt(value, 10, 64); err == nil { - size = parsed - } - } - return manifestOutcome{manifest: &cachedContainerManifest{ - contentType: resp.Header.Get(headerContentType), - contentDigest: digest, - etag: resp.Header.Get(headerETag), - size: size, - lastModified: parseHTTPTime(resp.Header.Get(headerLastModified)), - }} - case shouldServeStaleManifest(resp.StatusCode) && (cached != nil || referrersTagPattern.MatchString(req.reference)): - return manifestOutcome{err: fmt.Errorf("upstream status %d", resp.StatusCode)} - } - return manifestOutcome{relay: h.captureResponse(resp)} -} - -// digestMismatch reports whether two digests of the same algorithm differ. -// Digests of different algorithms can't be compared without the body. -func digestMismatch(requested, upstream string) bool { - reqAlg, reqHex, _ := strings.Cut(requested, ":") - upAlg, upHex, _ := strings.Cut(upstream, ":") - return reqAlg == upAlg && !strings.EqualFold(reqHex, upHex) -} - -// tagFromCachedDigest answers a tag from the cache once a HEAD named the -// digest it points to: the cached tag entry if the tag did not move, or the -// manifest stored under the new digest. It reports false when neither is -// cached. -func (h *ContainerHandler) tagFromCachedDigest(ctx context.Context, req manifestRequest, cached *cachedContainerManifest, fromFallback bool, resp *http.Response, digest string) (manifestOutcome, bool) { - if cached != nil && cached.contentDigest == digest { - cached.fetchedAt = time.Now() - h.refreshManifest(ctx, req, cached, fromFallback) - return manifestOutcome{manifest: cached}, true - } - - stored, err := h.loadContainerManifest(ctx, h.containerManifestSharedCacheKey(req.registryURL, req.name, digest, req.cacheAccept)) - if err != nil || stored == nil || stored.contentDigest != digest || !containerManifestCacheCompatible(req.accept, stored) { - return manifestOutcome{}, false - } - moved := *stored - moved.fetchedAt = time.Now() - if etag := resp.Header.Get(headerETag); etag != "" { - moved.etag = etag - } - if lastModified := parseHTTPTime(resp.Header.Get(headerLastModified)); !lastModified.IsZero() { - moved.lastModified = lastModified - } - h.storeContainerManifestForAccept(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept, &moved) - return manifestOutcome{manifest: &moved}, true -} - -// refreshManifest stores cached again with its new fetch time. -func (h *ContainerHandler) refreshManifest(ctx context.Context, req manifestRequest, cached *cachedContainerManifest, fromFallback bool) { - if fromFallback { - // The shared entry holds another representation for other - // clients; only this client's own entries are refreshed. - h.storeContainerManifestLegacyEntries(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept, cached) - return - } - h.storeContainerManifestForAccept(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept, cached) -} - -// getManifest GETs fetchRef, the requested reference or the digest a tag -// resolved to, and stores the manifest under the requested reference and -// its digest. cached, if set, is revalidated with its ETag. -func (h *ContainerHandler) getManifest(ctx context.Context, req manifestRequest, fetchRef string, cached *cachedContainerManifest, fromFallback bool) manifestOutcome { - etag := "" - if cached != nil { - etag = cached.etag - } - resp, err := h.requestManifest(ctx, http.MethodGet, req, fetchRef, etag) - if err != nil { - return manifestOutcome{err: err} - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode == http.StatusNotModified && cached != nil { - cached.fetchedAt = time.Now() - h.refreshManifest(ctx, req, cached, fromFallback) - return manifestOutcome{manifest: cached} - } - if resp.StatusCode != http.StatusOK { - if shouldServeStaleManifest(resp.StatusCode) && (cached != nil || referrersTagPattern.MatchString(req.reference)) { - return manifestOutcome{err: fmt.Errorf("upstream status %d", resp.StatusCode)} - } - return manifestOutcome{relay: h.captureResponse(resp)} - } - - body, err := h.proxy.ReadMetadata(resp.Body) - if err != nil { - return manifestOutcome{err: fmt.Errorf("reading manifest: %w", err)} - } - contentDigest := resp.Header.Get("Docker-Content-Digest") - // The references fetched and stored under, and the digest the upstream - // says the manifest has, are all cache keys, so each digest among them - // must be the manifest's own. One the proxy can't compute is refused - // rather than cached unchecked. - for _, expected := range []string{req.reference, fetchRef, contentDigest} { - if !strings.Contains(expected, ":") { - continue - } - if err := verifyDigest(expected, body); err != nil { - h.proxy.Logger.Error("upstream manifest failed digest verification", - "name", req.name, "reference", req.reference, "expected", expected, "error", err) - return manifestOutcome{digestInvalid: true} - } - } - if contentDigest == "" { - contentDigest = sha256Digest(body) - } - - manifest := &cachedContainerManifest{ - body: body, - contentType: resp.Header.Get(headerContentType), - contentDigest: contentDigest, - etag: resp.Header.Get(headerETag), - size: int64(len(body)), - lastModified: parseHTTPTime(resp.Header.Get(headerLastModified)), - fetchedAt: time.Now(), - } - h.storeContainerManifestForAccept(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept, manifest) - if manifest.contentDigest != req.reference && manifestDigestReferencePattern.MatchString(manifest.contentDigest) { - h.storeContainerManifestForAccept(ctx, req.registryURL, req.name, manifest.contentDigest, req.accept, req.cacheAccept, manifest) - } - return manifestOutcome{manifest: manifest} -} - -// requestManifest sends one manifest request upstream with the client's -// Accept header. The registry client adds credentials and follows bearer -// challenges for HEAD as for GET. -func (h *ContainerHandler) requestManifest(ctx context.Context, method string, req manifestRequest, reference, etag string) (*http.Response, error) { - upstream, err := http.NewRequestWithContext(ctx, method, req.url(reference), nil) - if err != nil { - return nil, fmt.Errorf("creating manifest request: %w", err) - } - upstream.Header.Set("Accept", req.accept) - if etag != "" { - upstream.Header.Set("If-None-Match", etag) - } - return h.proxy.HTTPClient.Do(upstream) -} diff --git a/internal/handler/container_manifest_key_test.go b/internal/handler/container_manifest_key_test.go deleted file mode 100644 index 75ad3d5..0000000 --- a/internal/handler/container_manifest_key_test.go +++ /dev/null @@ -1,243 +0,0 @@ -package handler - -import ( - "io" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" -) - -const ( - dockerManifestAccept = "application/vnd.docker.distribution.manifest.v2+json, " + - "application/vnd.docker.distribution.manifest.list.v2+json, " + - "application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json" - containerdManifestAccept = "application/vnd.docker.distribution.manifest.v2+json, " + - "application/vnd.docker.distribution.manifest.list.v2+json, " + - "application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" - ociManifestMediaType = "application/vnd.oci.image.manifest.v1+json" -) - -// fakeManifestRegistry serves one document per manifest path and fails with -// 503 once it is switched off. -type fakeManifestRegistry struct { - server *httptest.Server - offline atomic.Bool - docs map[string]fakeManifest -} - -type fakeManifest struct { - contentType string - body string -} - -func newFakeManifestRegistry(t *testing.T) *fakeManifestRegistry { - t.Helper() - registry := &fakeManifestRegistry{docs: make(map[string]fakeManifest)} - registry.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if registry.offline.Load() { - http.Error(w, "offline", http.StatusServiceUnavailable) - return - } - doc, ok := registry.docs[r.URL.Path] - if !ok { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", doc.contentType) - w.Header().Set("Docker-Content-Digest", sha256Digest([]byte(doc.body))) - _, _ = io.WriteString(w, doc.body) - })) - t.Cleanup(registry.server.Close) - return registry -} - -func (r *fakeManifestRegistry) set(path, contentType, body string) { - r.docs[path] = fakeManifest{contentType: contentType, body: body} -} - -func newManifestKeyTestHandler(t *testing.T, registry *fakeManifestRegistry) *ContainerHandler { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = registry.server.Client() - // A zero TTL makes every tag request revalidate, so an offline upstream - // exercises the stale path a real offline pull takes. - proxy.MetadataTTL = 0 - return &ContainerHandler{proxy: proxy, registryURL: registry.server.URL, proxyURL: "http://localhost:8080"} -} - -func getManifest(t *testing.T, h *ContainerHandler, path, accept string) *httptest.ResponseRecorder { - t.Helper() - request := httptest.NewRequest(http.MethodGet, path, nil) - request.Header.Set("Accept", accept) - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, request) - return response -} - -func TestContainerHandler_SingleArchTagSharedAcrossClients(t *testing.T) { - manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` - registry := newFakeManifestRegistry(t) - registry.set("/v2/library/app/manifests/1.0", ociManifestMediaType, manifest) - h := newManifestKeyTestHandler(t, registry) - - if response := getManifest(t, h, "/library/app/manifests/1.0", dockerManifestAccept); response.Code != http.StatusOK { - t.Fatalf("docker pull status = %d: %s", response.Code, response.Body.String()) - } - registry.offline.Store(true) - - response := getManifest(t, h, "/library/app/manifests/1.0", containerdManifestAccept) - if response.Code != http.StatusOK { - t.Fatalf("offline containerd pull status = %d, want 200: %s", response.Code, response.Body.String()) - } - if response.Body.String() != manifest { - t.Errorf("body = %q, want %q", response.Body.String(), manifest) - } -} - -func TestContainerHandler_DigestManifestSharedAcrossClients(t *testing.T) { - manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{}}` - digest := sha256Digest([]byte(manifest)) - registry := newFakeManifestRegistry(t) - registry.set("/v2/library/app/manifests/"+digest, ociManifestMediaType, manifest) - h := newManifestKeyTestHandler(t, registry) - - if response := getManifest(t, h, "/library/app/manifests/"+digest, dockerManifestAccept); response.Code != http.StatusOK { - t.Fatalf("docker pull status = %d: %s", response.Code, response.Body.String()) - } - registry.offline.Store(true) - - response := getManifest(t, h, "/library/app/manifests/"+digest, ociManifestMediaType) - if response.Code != http.StatusOK { - t.Fatalf("offline pull by digest status = %d, want 200: %s", response.Code, response.Body.String()) - } -} - -func TestContainerHandler_RepushedTagServesLatestDocument(t *testing.T) { - index := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[]}` - manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` - registry := newFakeManifestRegistry(t) - registry.set("/v2/library/app/manifests/latest", ociImageIndexMediaType, index) - h := newManifestKeyTestHandler(t, registry) - - if response := getManifest(t, h, "/library/app/manifests/latest", dockerManifestAccept); response.Body.String() != index { - t.Fatalf("first pull body = %q, want index", response.Body.String()) - } - registry.set("/v2/library/app/manifests/latest", ociManifestMediaType, manifest) - if response := getManifest(t, h, "/library/app/manifests/latest", containerdManifestAccept); response.Body.String() != manifest { - t.Fatalf("pull after re-push body = %q, want manifest", response.Body.String()) - } - registry.offline.Store(true) - - response := getManifest(t, h, "/library/app/manifests/latest", dockerManifestAccept) - if response.Code != http.StatusOK { - t.Fatalf("offline pull status = %d, want 200: %s", response.Code, response.Body.String()) - } - if response.Body.String() != manifest { - t.Errorf("offline body = %q, want the re-pushed manifest", response.Body.String()) - } -} - -func TestContainerHandler_IndexlessClientKeepsOwnTagEntry(t *testing.T) { - index := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[]}` - converted := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json"}` - const schema2 = "application/vnd.docker.distribution.manifest.v2+json" - registry := newFakeManifestRegistry(t) - registry.set("/v2/library/app/manifests/latest", schema2, converted) - h := newManifestKeyTestHandler(t, registry) - - // A client without index support gets the converted platform manifest. - if response := getManifest(t, h, "/library/app/manifests/latest", schema2); response.Body.String() != converted { - t.Fatalf("index-less pull body = %q", response.Body.String()) - } - registry.set("/v2/library/app/manifests/latest", ociImageIndexMediaType, index) - if response := getManifest(t, h, "/library/app/manifests/latest", dockerManifestAccept); response.Body.String() != index { - t.Fatalf("index-capable pull body = %q, want the index", response.Body.String()) - } - registry.offline.Store(true) - - if response := getManifest(t, h, "/library/app/manifests/latest", containerdManifestAccept); response.Body.String() != index { - t.Errorf("offline index-capable body = %q, want the index", response.Body.String()) - } - if response := getManifest(t, h, "/library/app/manifests/latest", schema2); response.Body.String() != converted { - t.Errorf("offline index-less body = %q, want the converted manifest", response.Body.String()) - } -} - -func TestContainerHandler_IncompatibleSharedEntryFallsBackToAcceptEntry(t *testing.T) { - list := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.list.v2+json","manifests":[]}` - index := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[]}` - registry := newFakeManifestRegistry(t) - registry.set("/v2/library/app/manifests/latest", dockerManifestListMediaType, list) - h := newManifestKeyTestHandler(t, registry) - - const dockerOnly = "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json" - if response := getManifest(t, h, "/library/app/manifests/latest", dockerOnly); response.Body.String() != list { - t.Fatalf("Docker-only pull body = %q, want the manifest list", response.Body.String()) - } - registry.set("/v2/library/app/manifests/latest", ociImageIndexMediaType, index) - if response := getManifest(t, h, "/library/app/manifests/latest", ociImageIndexMediaType); response.Body.String() != index { - t.Fatalf("OCI pull body = %q, want the index", response.Body.String()) - } - registry.offline.Store(true) - - response := getManifest(t, h, "/library/app/manifests/latest", dockerOnly) - if response.Code != http.StatusOK { - t.Fatalf("offline Docker-only pull status = %d, want 200: %s", response.Code, response.Body.String()) - } - if response.Body.String() != list { - t.Errorf("offline Docker-only body = %q, want the manifest list", response.Body.String()) - } - if response.Header().Get("Warning") == "" { - t.Error("offline Docker-only response has no stale warning, so it did not come from the stale path") - } - // Serving the fallback entry must not copy it over the shared entry, - // which still holds the index for clients that accept both. - if response := getManifest(t, h, "/library/app/manifests/latest", dockerManifestAccept); response.Body.String() != index { - t.Errorf("offline body for a client accepting both = %q, want the index still in the shared entry", response.Body.String()) - } -} - -func TestContainerHandler_NotModifiedFallbackKeepsSharedEntry(t *testing.T) { - list := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.list.v2+json","manifests":[]}` - index := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[]}` - var offline atomic.Bool - // The registry answers per Accept header, with ETags, like registries - // that convert OCI indexes for clients without OCI support. - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if offline.Load() { - http.Error(w, "offline", http.StatusServiceUnavailable) - return - } - body, contentType := list, dockerManifestListMediaType - if strings.Contains(r.Header.Get("Accept"), ociImageIndexMediaType) { - body, contentType = index, ociImageIndexMediaType - } - etag := `"` + sha256Digest([]byte(body)) + `"` - if r.Header.Get("If-None-Match") == etag { - w.WriteHeader(http.StatusNotModified) - return - } - w.Header().Set("Content-Type", contentType) - w.Header().Set("ETag", etag) - _, _ = io.WriteString(w, body) - })) - t.Cleanup(upstream.Close) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - - const dockerOnly = "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json" - getManifest(t, h, "/library/app/manifests/latest", dockerOnly) - getManifest(t, h, "/library/app/manifests/latest", ociImageIndexMediaType) - // Revalidates the Docker-only fallback entry and gets 304. - if response := getManifest(t, h, "/library/app/manifests/latest", dockerOnly); response.Body.String() != list { - t.Fatalf("Docker-only revalidation body = %q, want the list", response.Body.String()) - } - offline.Store(true) - - if response := getManifest(t, h, "/library/app/manifests/latest", dockerManifestAccept); response.Body.String() != index { - t.Errorf("body for a client accepting both = %q, want the index; the 304 refresh overwrote the shared entry", response.Body.String()) - } -} diff --git a/internal/handler/container_manifest_revalidate_test.go b/internal/handler/container_manifest_revalidate_test.go deleted file mode 100644 index abe0fc5..0000000 --- a/internal/handler/container_manifest_revalidate_test.go +++ /dev/null @@ -1,422 +0,0 @@ -package handler - -import ( - "io" - "net/http" - "net/http/httptest" - "strconv" - "strings" - "sync" - "testing" - "time" -) - -const revalidateRepo = "/v2/library/app/manifests/" - -// countingRegistry serves manifests by tag and digest and counts GET and HEAD -// requests per reference, since Docker Hub counts only GETs as pulls. -type countingRegistry struct { - server *httptest.Server - - mu sync.Mutex - docs map[string]string // by digest - tags map[string]string // tag -> digest - gets map[string]int - heads map[string]int - headAccept string - // headDigest is false for a registry whose HEAD has no - // Docker-Content-Digest. - headDigest bool - // hold delays each response until holdFor requests arrived, or - // holdTimeout passed, so concurrent callers overlap upstream. - holdFor int - holdTimeout time.Duration - arrived int - release chan struct{} -} - -func newCountingRegistry(t *testing.T) *countingRegistry { - t.Helper() - registry := &countingRegistry{ - docs: make(map[string]string), - tags: make(map[string]string), - gets: make(map[string]int), - heads: make(map[string]int), - headDigest: true, - release: make(chan struct{}), - } - registry.server = httptest.NewServer(http.HandlerFunc(registry.serve)) - t.Cleanup(registry.server.Close) - return registry -} - -func (c *countingRegistry) serve(w http.ResponseWriter, r *http.Request) { - reference, ok := strings.CutPrefix(r.URL.Path, revalidateRepo) - if !ok { - http.NotFound(w, r) - return - } - c.mu.Lock() - if r.Method == http.MethodHead { - c.heads[reference]++ - c.headAccept = r.Header.Get("Accept") - } else { - c.gets[reference]++ - } - hold := c.holdFor > 0 - if hold { - c.arrived++ - if c.arrived == c.holdFor { - close(c.release) - } - } - digest, isTag := c.tags[reference] - if !isTag { - digest = reference - } - body, found := c.docs[digest] - headDigest := c.headDigest - c.mu.Unlock() - - if hold { - select { - case <-c.release: - case <-time.After(c.holdTimeout): - } - } - if !found { - http.NotFound(w, r) - return - } - etag := `"` + digest + `"` - w.Header().Set("Content-Type", ociManifestMediaType) - w.Header().Set("ETag", etag) - if r.Method != http.MethodHead || headDigest { - w.Header().Set("Docker-Content-Digest", digest) - } - if r.Header.Get("If-None-Match") == etag { - w.WriteHeader(http.StatusNotModified) - return - } - w.Header().Set("Content-Length", strconv.Itoa(len(body))) - if r.Method != http.MethodHead { - _, _ = io.WriteString(w, body) - } -} - -// add stores a manifest and returns its digest. -func (c *countingRegistry) add(body string) string { - digest := sha256Digest([]byte(body)) - c.mu.Lock() - c.docs[digest] = body - c.mu.Unlock() - return digest -} - -// pointLatest points the latest tag at digest. -func (c *countingRegistry) pointLatest(digest string) { - c.mu.Lock() - c.tags["latest"] = digest - c.mu.Unlock() -} - -// counts returns the GETs and HEADs seen so far, all references together. -func (c *countingRegistry) counts() (gets, heads int) { - c.mu.Lock() - defer c.mu.Unlock() - for _, n := range c.gets { - gets += n - } - for _, n := range c.heads { - heads += n - } - return gets, heads -} - -func (c *countingRegistry) getsOf(reference string) int { - c.mu.Lock() - defer c.mu.Unlock() - return c.gets[reference] -} - -func newRevalidateTestHandler(t *testing.T, registry *countingRegistry) *ContainerHandler { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = registry.server.Client() - proxy.MetadataTTL = time.Hour - return &ContainerHandler{proxy: proxy, registryURL: registry.server.URL, proxyURL: "http://localhost:8080"} -} - -// ageLatest moves the cached entry of the latest tag past the TTL. -func ageLatest(t *testing.T, h *ContainerHandler) { - t.Helper() - key := h.containerManifestSharedCacheKey(h.registryURL, "library/app", "latest", normalizeContainerManifestAccept(dockerManifestAccept)) - entry, err := h.proxy.DB.GetMetadataCache(containerManifestCacheEcosystem, key) - if err != nil || entry == nil { - t.Fatalf("cached tag row = %v, %v, want one", entry, err) - } - entry.FetchedAt.Time = time.Now().Add(-2 * time.Hour) - if err := h.proxy.DB.UpsertMetadataCache(entry); err != nil { - t.Fatalf("ageing cached tag: %v", err) - } -} - -func pullManifest(t *testing.T, h *ContainerHandler, method, reference string) *httptest.ResponseRecorder { - t.Helper() - request := httptest.NewRequest(method, "/library/app/manifests/"+reference, nil) - request.Header.Set("Accept", dockerManifestAccept) - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, request) - if response.Code != http.StatusOK { - t.Fatalf("%s %s status = %d: %s", method, reference, response.Code, response.Body.String()) - } - return response -} - -func manifestDoc(n string) string { - return `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","annotations":{"n":"` + n + `"}}` -} - -func TestContainerHandler_StaleTagUnchangedRevalidatesWithHead(t *testing.T) { - registry := newCountingRegistry(t) - digest := registry.add(manifestDoc("a")) - registry.pointLatest(digest) - h := newRevalidateTestHandler(t, registry) - - pullManifest(t, h, http.MethodGet, "latest") - ageLatest(t, h) - response := pullManifest(t, h, http.MethodGet, "latest") - - if response.Body.String() != manifestDoc("a") { - t.Errorf("body = %q, want the cached manifest", response.Body.String()) - } - if gets, heads := registry.counts(); gets != 1 || heads != 1 { - t.Errorf("upstream GETs = %d, HEADs = %d; want 1 GET for the first pull and 1 HEAD to revalidate", gets, heads) - } - registry.mu.Lock() - headAccept := registry.headAccept - registry.mu.Unlock() - if headAccept != dockerManifestAccept { - t.Errorf("HEAD Accept = %q, want the client's %q", headAccept, dockerManifestAccept) - } - // The HEAD refreshed the entry, so the next pull stays local. - pullManifest(t, h, http.MethodGet, "latest") - if gets, heads := registry.counts(); gets != 1 || heads != 1 { - t.Errorf("after a fresh pull: GETs = %d, HEADs = %d, want 1 and 1", gets, heads) - } -} - -func TestContainerHandler_StaleTagMovedToCachedDigestNeedsNoGet(t *testing.T) { - registry := newCountingRegistry(t) - first := registry.add(manifestDoc("a")) - second := registry.add(manifestDoc("b")) - registry.pointLatest(first) - h := newRevalidateTestHandler(t, registry) - - pullManifest(t, h, http.MethodGet, "latest") - pullManifest(t, h, http.MethodGet, second) - registry.pointLatest(second) - ageLatest(t, h) - - response := pullManifest(t, h, http.MethodGet, "latest") - if response.Body.String() != manifestDoc("b") { - t.Errorf("body = %q, want the manifest the tag moved to", response.Body.String()) - } - if got := response.Header().Get("Docker-Content-Digest"); got != second { - t.Errorf("Docker-Content-Digest = %q, want %q", got, second) - } - if gets, heads := registry.counts(); gets != 2 || heads != 1 { - t.Errorf("upstream GETs = %d, HEADs = %d; want only the 2 setup GETs and 1 HEAD", gets, heads) - } -} - -func TestContainerHandler_StaleTagMovedToNewDigestGetsByDigest(t *testing.T) { - registry := newCountingRegistry(t) - first := registry.add(manifestDoc("a")) - registry.pointLatest(first) - h := newRevalidateTestHandler(t, registry) - - pullManifest(t, h, http.MethodGet, "latest") - second := registry.add(manifestDoc("b")) - registry.pointLatest(second) - ageLatest(t, h) - - response := pullManifest(t, h, http.MethodGet, "latest") - if response.Body.String() != manifestDoc("b") { - t.Errorf("body = %q, want the manifest the tag moved to", response.Body.String()) - } - if got := registry.getsOf("latest"); got != 1 { - t.Errorf("GETs of the tag = %d, want only the first pull's", got) - } - if got := registry.getsOf(second); got != 1 { - t.Errorf("GETs of the new digest = %d, want 1", got) - } - if _, heads := registry.counts(); heads != 1 { - t.Errorf("HEADs = %d, want 1", heads) - } - // Both the tag and the digest are now cached. - pullManifest(t, h, http.MethodGet, "latest") - pullManifest(t, h, http.MethodGet, second) - if gets, heads := registry.counts(); gets != 2 || heads != 1 { - t.Errorf("after cached pulls: GETs = %d, HEADs = %d, want 2 and 1", gets, heads) - } -} - -func TestContainerHandler_ConcurrentStaleTagRevalidatesOnce(t *testing.T) { - const pulls = 8 - registry := newCountingRegistry(t) - digest := registry.add(manifestDoc("a")) - registry.pointLatest(digest) - h := newRevalidateTestHandler(t, registry) - - pullManifest(t, h, http.MethodGet, "latest") - ageLatest(t, h) - setupGets, setupHeads := registry.counts() - registry.mu.Lock() - // Hold upstream answers until every pull reached upstream, or long - // enough that a coalesced request is the only one. - registry.holdFor = pulls - registry.holdTimeout = 300 * time.Millisecond - registry.mu.Unlock() - - var wg sync.WaitGroup - for range pulls { - wg.Go(func() { - request := httptest.NewRequest(http.MethodGet, "/library/app/manifests/latest", nil) - request.Header.Set("Accept", dockerManifestAccept) - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, request) - if response.Code != http.StatusOK || response.Body.String() != manifestDoc("a") { - t.Errorf("concurrent pull = %d %q", response.Code, response.Body.String()) - } - }) - } - wg.Wait() - - gets, heads := registry.counts() - if requests := gets - setupGets + heads - setupHeads; requests != 1 { - t.Errorf("upstream requests for %d concurrent pulls = %d, want 1", pulls, requests) - } -} - -func TestContainerHandler_StaleTagHeadWithoutDigestFallsBackToGet(t *testing.T) { - registry := newCountingRegistry(t) - first := registry.add(manifestDoc("a")) - registry.pointLatest(first) - registry.headDigest = false - h := newRevalidateTestHandler(t, registry) - - pullManifest(t, h, http.MethodGet, "latest") - second := registry.add(manifestDoc("b")) - registry.pointLatest(second) - ageLatest(t, h) - - response := pullManifest(t, h, http.MethodGet, "latest") - if response.Body.String() != manifestDoc("b") { - t.Errorf("body = %q, want the manifest the tag moved to", response.Body.String()) - } - if got := registry.getsOf("latest"); got != 2 { - t.Errorf("GETs of the tag = %d, want 2: the HEAD gave no digest", got) - } -} - -func TestContainerHandler_ClientHeadPointsTagAtCachedDigest(t *testing.T) { - registry := newCountingRegistry(t) - first := registry.add(manifestDoc("a")) - second := registry.add(manifestDoc("b")) - registry.pointLatest(first) - h := newRevalidateTestHandler(t, registry) - - pullManifest(t, h, http.MethodGet, "latest") - pullManifest(t, h, http.MethodGet, second) - registry.pointLatest(second) - ageLatest(t, h) - - head := pullManifest(t, h, http.MethodHead, "latest") - if got := head.Header().Get("Docker-Content-Digest"); got != second { - t.Errorf("HEAD Docker-Content-Digest = %q, want %q", got, second) - } - // The HEAD moved the cached tag, so the GET that follows it in a pull - // is a cache hit. - gets, heads := registry.counts() - response := pullManifest(t, h, http.MethodGet, "latest") - if response.Body.String() != manifestDoc("b") { - t.Errorf("body = %q, want the manifest the tag moved to", response.Body.String()) - } - if afterGets, afterHeads := registry.counts(); afterGets != gets || afterHeads != heads { - t.Errorf("GET after HEAD went upstream: GETs %d -> %d, HEADs %d -> %d", gets, afterGets, heads, afterHeads) - } -} - -func TestContainerHandler_ClientHeadOfUncachedDigestIsNotStored(t *testing.T) { - registry := newCountingRegistry(t) - first := registry.add(manifestDoc("a")) - registry.pointLatest(first) - h := newRevalidateTestHandler(t, registry) - - pullManifest(t, h, http.MethodGet, "latest") - second := registry.add(manifestDoc("b")) - registry.pointLatest(second) - ageLatest(t, h) - - head := pullManifest(t, h, http.MethodHead, "latest") - if got := head.Header().Get("Docker-Content-Digest"); got != second { - t.Errorf("HEAD Docker-Content-Digest = %q, want %q", got, second) - } - if got := head.Header().Get("Content-Length"); got != strconv.Itoa(len(manifestDoc("b"))) { - t.Errorf("HEAD Content-Length = %q, want the upstream manifest size", got) - } - if gets, _ := registry.counts(); gets != 1 { - t.Errorf("GETs = %d, want 1: a client HEAD must not cost a pull", gets) - } -} - -func TestContainerHandler_ConcurrentDigestMissFetchesOnce(t *testing.T) { - const pulls = 8 - registry := newCountingRegistry(t) - digest := registry.add(manifestDoc("a")) - registry.holdFor = pulls - registry.holdTimeout = 300 * time.Millisecond - h := newRevalidateTestHandler(t, registry) - - var wg sync.WaitGroup - for range pulls { - wg.Go(func() { - request := httptest.NewRequest(http.MethodGet, "/library/app/manifests/"+digest, nil) - request.Header.Set("Accept", dockerManifestAccept) - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, request) - if response.Code != http.StatusOK || response.Body.String() != manifestDoc("a") { - t.Errorf("concurrent pull = %d %q", response.Code, response.Body.String()) - } - }) - } - wg.Wait() - - if got := registry.getsOf(digest); got != 1 { - t.Errorf("upstream GETs for %d concurrent pulls by digest = %d, want 1", pulls, got) - } -} - -func TestContainerHandler_StaleTagDeletedUpstreamRelaysNotFound(t *testing.T) { - registry := newCountingRegistry(t) - registry.pointLatest(registry.add(manifestDoc("a"))) - h := newRevalidateTestHandler(t, registry) - - pullManifest(t, h, http.MethodGet, "latest") - registry.mu.Lock() - delete(registry.tags, "latest") - registry.mu.Unlock() - ageLatest(t, h) - - request := httptest.NewRequest(http.MethodGet, "/library/app/manifests/latest", nil) - request.Header.Set("Accept", dockerManifestAccept) - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, request) - if response.Code != http.StatusNotFound { - t.Fatalf("status = %d, want the upstream 404", response.Code) - } - if !strings.Contains(response.Body.String(), "not found") { - t.Errorf("body = %q, want the upstream error body", response.Body.String()) - } -} diff --git a/internal/handler/container_metadata.go b/internal/handler/container_metadata.go deleted file mode 100644 index 9e1b24a..0000000 --- a/internal/handler/container_metadata.go +++ /dev/null @@ -1,62 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "database/sql" - "fmt" - "time" - - "github.com/git-pkgs/proxy/internal/database" -) - -func (h *ContainerHandler) storeContainerMetadata(ctx context.Context, ecosystem, cacheKey string, body []byte, etag, link, contentType, contentDigest string, lastModified, fetchedAt time.Time) (int64, error) { - if h.proxy.DB == nil || h.proxy.Storage == nil { - return int64(len(body)), nil - } - - storagePath := metadataStoragePath(ecosystem, cacheKey) - size, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(body)) - if err != nil { - return 0, fmt.Errorf("storing metadata: %w", err) - } - err = h.proxy.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ - Ecosystem: ecosystem, - Name: cacheKey, - StoragePath: storagePath, - ETag: sql.NullString{String: etag, Valid: etag != ""}, - Link: sql.NullString{String: link, Valid: link != ""}, - ContentType: sql.NullString{String: contentType, Valid: contentType != ""}, - ContentDigest: sql.NullString{String: contentDigest, Valid: contentDigest != ""}, - Size: sql.NullInt64{Int64: size, Valid: true}, - LastModified: sql.NullTime{Time: lastModified, Valid: !lastModified.IsZero()}, - FetchedAt: sql.NullTime{Time: fetchedAt, Valid: !fetchedAt.IsZero()}, - }) - if err != nil { - return 0, err - } - return size, nil -} - -// loadContainerMetadata returns a cached metadata row and its body, or nil -// when nothing usable is cached. -func (h *ContainerHandler) loadContainerMetadata(ctx context.Context, ecosystem, cacheKey string) (*database.MetadataCacheEntry, []byte, error) { - if h.proxy.DB == nil || h.proxy.Storage == nil { - return nil, nil, nil - } - entry, err := h.proxy.DB.GetMetadataCache(ecosystem, cacheKey) - if err != nil || entry == nil { - return nil, nil, err - } - reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) - if err != nil { - return nil, nil, nil - } - defer func() { _ = reader.Close() }() - body, err := h.proxy.ReadMetadata(reader) - if err != nil { - return nil, nil, err - } - h.proxy.recordMetadataRead(entry) - return entry, body, nil -} diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go deleted file mode 100644 index f19ac73..0000000 --- a/internal/handler/container_mirror.go +++ /dev/null @@ -1,549 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "regexp" - "slices" - "strings" - "sync" - "sync/atomic" - "time" - - "golang.org/x/sync/errgroup" -) - -const ( - // maxMirrorIndexDepth bounds how deep nested indexes are followed. - maxMirrorIndexDepth = 2 - // mirrorErrorBodyLimit bounds how much of an error response is quoted. - mirrorErrorBodyLimit = 200 - // mirrorBlobConcurrency is how many blobs of one manifest are fetched at - // once, as many as a Docker pull fetches by default. - mirrorBlobConcurrency = 3 - - dockerReferenceTypeAnnotation = "vnd.docker.reference.type" - dockerReferenceDigestAnnotation = "vnd.docker.reference.digest" - attestationManifestReference = "attestation-manifest" -) - -var ( - // Grammar from the OCI distribution spec. - mirrorNamePattern = regexp.MustCompile(`^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*$`) - mirrorTagPattern = regexp.MustCompile(`^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$`) - mirrorDigestPattern = regexp.MustCompile(`^(sha256:[a-f0-9]{64}|sha512:[a-f0-9]{128})$`) -) - -// dockerHubMirrorAliases are registry hosts that name Docker Hub in image -// references. hub.docker.com is the default registry of the docker PURL type. -var dockerHubMirrorAliases = []string{"docker.io", "index.docker.io", "registry-1.docker.io", "hub.docker.com"} //nolint:gochecknoglobals // fixed alias list - -// ImageRef names an image to mirror. -type ImageRef struct { - // Registry is the registry host the image reference names, as containerd - // sends it in the ns query parameter. Empty routes Name like a pull - // without ns. - Registry string - // Name is the repository name. - Name string - // References are the tags and digests to mirror. - References []string -} - -// Platform selects the entries of an image index to mirror. Empty fields -// match any value. -type Platform struct { - OS string - Architecture string - Variant string -} - -// ParsePlatform parses os/architecture[/variant]. -func ParsePlatform(value string) (Platform, error) { - os, rest, _ := strings.Cut(value, "/") - architecture, variant, _ := strings.Cut(rest, "/") - if os == "" || architecture == "" || strings.Contains(variant, "/") { - return Platform{}, fmt.Errorf("invalid platform %q, want os/architecture[/variant]", value) - } - return Platform{OS: os, Architecture: architecture, Variant: variant}, nil -} - -func (p Platform) matches(entry *ociPlatform) bool { - if entry == nil { - return false - } - if p.OS != "" && p.OS != entry.OS { - return false - } - if p.Architecture != "" && p.Architecture != entry.Architecture { - return false - } - return p.Variant == "" || p.Variant == platformVariant(entry) -} - -// platformVariant fills in the variant arm64 images usually leave out. -func platformVariant(entry *ociPlatform) string { - if entry.Variant == "" && entry.Architecture == "arm64" { - return "v8" - } - return entry.Variant -} - -// ImageResult reports what mirroring one image did. -type ImageResult struct { - // Fetched counts manifests and blobs that were not cached before. - Fetched int - // Bytes is the size of the blobs fetched for this image. - Bytes int64 - Warnings []string -} - -type ociDescriptor struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - URLs []string `json:"urls"` - Platform *ociPlatform `json:"platform"` - Annotations map[string]string `json:"annotations"` -} - -type ociPlatform struct { - OS string `json:"os"` - Architecture string `json:"architecture"` - Variant string `json:"variant"` -} - -type ociManifest struct { - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` - Manifests []ociDescriptor `json:"manifests"` - Config *ociDescriptor `json:"config"` - Layers []ociDescriptor `json:"layers"` -} - -type imageMirror struct { - h *ContainerHandler - ref ImageRef - registryURL string - upstreamName string - cacheName string - platforms []Platform - seen map[string]bool - result ImageResult - // platformMatches counts index entries that matched the platform filter - // during the current walk. - platformMatches int -} - -// MirrorImage fetches an image the way a client pull does: the manifest or -// index for each reference, the platform manifests, then config and layer -// blobs. Everything is stored under the keys a later pull through this handler -// reads, so the pull is served from cache. Platforms limit which index entries -// are followed; without them every platform is mirrored. -func (h *ContainerHandler) MirrorImage(ctx context.Context, ref ImageRef, platforms []Platform) (ImageResult, error) { - ref, err := normalizeImageRef(ref, h.registryURL) - if err != nil { - return ImageResult{}, err - } - route := &http.Request{URL: &url.URL{RawQuery: url.Values{namespaceQueryParam: {ref.Registry}}.Encode()}} - if ref.Registry == "" { - route.URL.RawQuery = "" - } - registryURL, upstreamName, cacheName, ok := h.registryForRequest(route, ref.Name) - if !ok { - if ref.Registry != "" { - return ImageResult{}, fmt.Errorf("registry %q is not configured as an OCI upstream", ref.Registry) - } - return ImageResult{}, fmt.Errorf("no OCI upstream for %q", ref.Name) - } - m := &imageMirror{ - h: h, - ref: ref, - registryURL: registryURL, - upstreamName: upstreamName, - cacheName: cacheName, - platforms: platforms, - seen: make(map[string]bool), - } - var digests []string - for _, reference := range ref.References { - mirror := m.mirrorTag - if mirrorDigestPattern.MatchString(reference) { - mirror = func(ctx context.Context, digest string) (string, error) { return m.mirrorManifest(ctx, digest, 0) } - } - digest, err := mirror(ctx, reference) - if err != nil { - return m.result, fmt.Errorf("%s:%s: %w", ref.Name, reference, err) - } - digests = append(digests, digest) - } - if len(digests) == 2 && digests[0] != digests[1] && mirrorDigestPattern.MatchString(ref.References[0]) { - m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("tag %s now points to %s, not %s; both were mirrored", - ref.References[1], digests[1], ref.References[0])) - } - return m.result, nil -} - -// normalizeImageRef validates the reference before any network access and -// adds the library/ prefix Docker Hub clients use for official images. -func normalizeImageRef(ref ImageRef, defaultRegistryURL string) (ImageRef, error) { - if slices.Contains(dockerHubMirrorAliases, strings.ToLower(ref.Registry)) { - ref.Registry = "docker.io" - } - dockerHub := ref.Registry == "docker.io" || (ref.Registry == "" && defaultRegistryURL == dockerHubRegistry) - if dockerHub && !strings.Contains(ref.Name, "/") { - ref.Name = "library/" + ref.Name - } - if !mirrorNamePattern.MatchString(ref.Name) { - return ref, fmt.Errorf("invalid repository name %q", ref.Name) - } - if len(ref.References) == 0 { - return ref, errors.New("a tag or digest is required; mirroring all tags is not supported") - } - for _, reference := range ref.References { - if !mirrorTagPattern.MatchString(reference) && !mirrorDigestPattern.MatchString(reference) { - return ref, fmt.Errorf("invalid tag or digest %q", reference) - } - } - return ref, nil -} - -// errTagUpstreamUnavailable marks tag lookups that failed because the -// upstream could not be reached or answered with 429 or 5xx. -var errTagUpstreamUnavailable = errors.New("upstream unavailable") - -// mirrorTag resolves a tag upstream without caching it, mirrors the manifest -// it names by digest, and stores the tag only after that manifest and -// everything it references are cached. A failed mirror leaves the previously -// cached tag as it was. If the upstream is unreachable, the cached tag is -// mirrored as it is. -func (m *imageMirror) mirrorTag(ctx context.Context, tag string) (string, error) { - accept := containerManifestAccept(&http.Request{Header: http.Header{}}) - cacheAccept := normalizeContainerManifestAccept(accept) - before, _ := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, tag, accept, cacheAccept) - - manifest, err := m.resolveTag(ctx, tag, accept, before) - if err != nil { - if before == nil || !errors.Is(err, errTagUpstreamUnavailable) { - return "", fmt.Errorf("manifest %s: %w", tag, err) - } - m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("manifest %s served from stale cache, upstream unreachable", tag)) - _, err := m.mirrorManifest(ctx, before.contentDigest, 0) - return before.contentDigest, err - } - - // The digest entry is immutable, so it can be stored right away; the - // walk below then reads it from cache instead of fetching it again. - digest := manifest.contentDigest - if before == nil || before.contentDigest != digest { - m.result.Fetched++ - m.h.storeContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, digest, accept, cacheAccept, manifest) - } - if _, err := m.mirrorManifest(ctx, digest, 0); err != nil { - return "", err - } - - // A client pull may have moved the tag while the walk ran. Its entry is - // newer than what this mirror resolved, so it is kept. - current, _ := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, tag, accept, cacheAccept) - if current != nil && current.contentDigest != digest && (before == nil || current.contentDigest != before.contentDigest) { - m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("tag %s moved to %s while it was mirrored; kept that entry", tag, current.contentDigest)) - return digest, nil - } - m.h.storeContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, tag, accept, cacheAccept, manifest) - return digest, nil -} - -// resolveTag returns the manifest a tag points to upstream without caching -// it. A HEAD request comes first: if the tag still points to the cached -// digest, the cached manifest is reused with a new fetch time, which spares a -// GET (registries such as Docker Hub do not count HEAD requests against pull -// limits). -func (m *imageMirror) resolveTag(ctx context.Context, tag, accept string, before *cachedContainerManifest) (*cachedContainerManifest, error) { - if before != nil { - response, err := m.requestTag(ctx, http.MethodHead, tag, accept) - if err != nil { - return nil, err - } - _ = response.Body.Close() - if response.Header.Get("Docker-Content-Digest") == before.contentDigest { - unchanged := *before - unchanged.fetchedAt = time.Now() - return &unchanged, nil - } - } - - response, err := m.requestTag(ctx, http.MethodGet, tag, accept) - if err != nil { - return nil, err - } - defer func() { _ = response.Body.Close() }() - body, err := m.h.proxy.ReadMetadata(response.Body) - if err != nil { - if ctx.Err() != nil { - return nil, ctx.Err() - } - return nil, fmt.Errorf("%w: reading manifest: %w", errTagUpstreamUnavailable, err) - } - digest := sha256Digest(body) - if contentDigest := response.Header.Get("Docker-Content-Digest"); strings.HasPrefix(contentDigest, "sha256:") && contentDigest != digest { - return nil, fmt.Errorf("digest verification failed: upstream says %s, body is %s", contentDigest, digest) - } - return &cachedContainerManifest{ - body: body, - contentType: response.Header.Get(headerContentType), - contentDigest: digest, - etag: response.Header.Get(headerETag), - size: int64(len(body)), - lastModified: parseHTTPTime(response.Header.Get(headerLastModified)), - fetchedAt: time.Now(), - }, nil -} - -// requestTag sends a manifest request for a tag upstream. Only 200 responses -// are returned; network errors, 429 and 5xx wrap errTagUpstreamUnavailable. -func (m *imageMirror) requestTag(ctx context.Context, method, tag, accept string) (*http.Response, error) { - upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", m.registryURL, m.upstreamName, tag) - request, err := http.NewRequestWithContext(ctx, method, upstreamURL, nil) - if err != nil { - return nil, err - } - request.Header.Set("Accept", accept) - response, err := m.h.proxy.HTTPClient.Do(request) - if err != nil { - if ctx.Err() != nil { - return nil, ctx.Err() - } - return nil, fmt.Errorf("%w: %w", errTagUpstreamUnavailable, err) - } - if response.StatusCode != http.StatusOK { - _ = response.Body.Close() - if shouldServeStaleManifest(response.StatusCode) { - return nil, fmt.Errorf("%w: status %d", errTagUpstreamUnavailable, response.StatusCode) - } - return nil, fmt.Errorf("upstream status %d", response.StatusCode) - } - return response, nil -} - -// mirrorManifest stores the manifest for reference through the handler's own -// manifest route, then follows what it references. It returns the manifest -// sha256 digest. -func (m *imageMirror) mirrorManifest(ctx context.Context, reference string, depth int) (string, error) { - cacheAccept := normalizeContainerManifestAccept(containerManifestAccept(&http.Request{Header: http.Header{}})) - before, _ := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, reference, cacheAccept, cacheAccept) - - target := "/" + m.ref.Name + "/manifests/" + reference - if m.ref.Registry != "" { - target += "?" + url.Values{namespaceQueryParam: {m.ref.Registry}}.Encode() - } - request, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) - if err != nil { - return "", err - } - response := newMirrorResponse() - m.h.Routes().ServeHTTP(response, request) - if response.status != http.StatusOK { - return "", fmt.Errorf("manifest %s: status %d: %s", reference, response.status, response.errorText()) - } - if response.header.Get("Warning") != "" { - m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("manifest %s served from stale cache, upstream unreachable", reference)) - } - digest := response.header.Get("Docker-Content-Digest") - if !mirrorDigestPattern.MatchString(digest) { - digest = sha256Digest(response.body.Bytes()) - } - if before == nil || before.contentDigest != digest { - m.result.Fetched++ - } - // Tags resolve to sha256 digests, so a manifest asked for by another - // digest is tracked by its sha256 too. - digest = sha256Digest(response.body.Bytes()) - if m.seen[digest] { - return digest, nil - } - m.seen[digest] = true - - var manifest ociManifest - if err := json.Unmarshal(response.body.Bytes(), &manifest); err != nil { - return "", fmt.Errorf("manifest %s: %w", reference, err) - } - if manifest.SchemaVersion == 1 { - return "", fmt.Errorf("manifest %s: schema 1 manifests are not supported", reference) - } - if manifest.Manifests == nil { - return digest, m.mirrorBlobs(ctx, &manifest) - } - if depth == 0 { - m.platformMatches = 0 - } - if err := m.mirrorIndex(ctx, reference, &manifest, depth); err != nil { - return "", err - } - // A nested index may lack the wanted platforms as long as the image as a - // whole has them. - if depth == 0 && len(m.platforms) > 0 && m.platformMatches == 0 { - return "", fmt.Errorf("index %s: no manifest for the requested platforms", reference) - } - return digest, nil -} - -// mirrorIndex follows the index entries for the wanted platforms. Docker -// fetches the attestation manifests of an index on every pull, so those of the -// selected platforms are mirrored too. -func (m *imageMirror) mirrorIndex(ctx context.Context, reference string, index *ociManifest, depth int) error { - if depth >= maxMirrorIndexDepth { - return fmt.Errorf("index %s: nested deeper than %d levels", reference, maxMirrorIndexDepth) - } - var selected, attestations []ociDescriptor - selectedDigests := make(map[string]bool) - for _, entry := range index.Manifests { - if !mirrorDigestPattern.MatchString(entry.Digest) { - return fmt.Errorf("index %s: invalid digest %q", reference, entry.Digest) - } - if entry.Annotations[dockerReferenceTypeAnnotation] == attestationManifestReference { - attestations = append(attestations, entry) - continue - } - if m.wantsPlatform(entry.Platform) { - selected = append(selected, entry) - selectedDigests[entry.Digest] = true - if entry.Platform != nil { - m.platformMatches++ - } - } - } - for _, entry := range attestations { - if len(m.platforms) == 0 || selectedDigests[entry.Annotations[dockerReferenceDigestAnnotation]] { - selected = append(selected, entry) - } - } - for _, entry := range selected { - if _, err := m.mirrorManifest(ctx, entry.Digest, depth+1); err != nil { - return err - } - } - return nil -} - -func (m *imageMirror) wantsPlatform(platform *ociPlatform) bool { - if len(m.platforms) == 0 || platform == nil { - // Entries without a platform are nested indexes or artifacts; a - // filter cannot judge them, so they are followed. - return true - } - for _, want := range m.platforms { - if want.matches(platform) { - return true - } - } - return false -} - -func (m *imageMirror) mirrorBlobs(ctx context.Context, manifest *ociManifest) error { - blobs := manifest.Layers - if manifest.Config != nil { - blobs = append([]ociDescriptor{*manifest.Config}, blobs...) - } - var digests []string - for _, blob := range blobs { - if len(blob.URLs) > 0 || strings.Contains(blob.MediaType, "nondistributable") || strings.Contains(blob.MediaType, "foreign") { - m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("skipped non-distributable layer %s", blob.Digest)) - continue - } - if !mirrorDigestPattern.MatchString(blob.Digest) { - return fmt.Errorf("invalid blob digest %q", blob.Digest) - } - if m.seen[blob.Digest] { - continue - } - m.seen[blob.Digest] = true - digests = append(digests, blob.Digest) - } - - // Large layers take most of the time, so they are fetched side by side. - // A failure starts no further blobs but lets those in flight finish - // rather than cancelling them: a client pull may share their fetch, and - // they would have to be fetched again next run. - var mu sync.Mutex - var failed atomic.Bool - var g errgroup.Group - g.SetLimit(mirrorBlobConcurrency) - for _, digest := range digests { - g.Go(func() error { - if failed.Load() { - return nil - } - size, fetched, err := m.mirrorBlob(ctx, digest) - if err != nil { - failed.Store(true) - return err - } - if !fetched { - return nil - } - mu.Lock() - m.result.Fetched++ - m.result.Bytes += size - mu.Unlock() - return nil - }) - } - return g.Wait() -} - -// mirrorBlob stores a blob under the same identity handleBlobDownload uses, -// and returns its size when it had to be fetched. -func (m *imageMirror) mirrorBlob(ctx context.Context, digest string) (int64, bool, error) { - proxy := m.h.proxy - cached, err := proxy.HasCachedArtifact(ctx, "oci", m.cacheName, digest, digest) - if err != nil { - return 0, false, fmt.Errorf("blob %s: %w", digest, err) - } - if cached { - // Not a cache hit, so LRU eviction still sees the image as unused, - // but retention keeps it while the schedule does. - if err := proxy.MarkMirrored("oci", m.cacheName, digest, digest); err != nil { - return 0, false, fmt.Errorf("blob %s: %w", digest, err) - } - return 0, false, nil - } - result, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "oci", m.cacheName, digest, digest, - fmt.Sprintf("%s/v2/%s/blobs/%s", m.registryURL, m.upstreamName, digest), digest) - if err != nil { - return 0, false, fmt.Errorf("blob %s: %w", digest, err) - } - if result.Reader != nil { - _ = result.Reader.Close() - } - return result.Artifact.Size, true, nil -} - -// mirrorResponse collects the handler's manifest response in memory. -type mirrorResponse struct { - header http.Header - status int - body bytes.Buffer -} - -func newMirrorResponse() *mirrorResponse { - return &mirrorResponse{header: make(http.Header), status: http.StatusOK} -} - -func (r *mirrorResponse) Header() http.Header { return r.header } - -func (r *mirrorResponse) WriteHeader(status int) { r.status = status } - -func (r *mirrorResponse) Write(p []byte) (int, error) { return r.body.Write(p) } - -func (r *mirrorResponse) errorText() string { - text := strings.TrimSpace(r.body.String()) - if len(text) > mirrorErrorBodyLimit { - text = text[:mirrorErrorBodyLimit] + "..." - } - return text -} diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go deleted file mode 100644 index 34af48c..0000000 --- a/internal/handler/container_mirror_test.go +++ /dev/null @@ -1,685 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha512" - "database/sql" - "encoding/hex" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "net/url" - "slices" - "strconv" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/registries/fetch" -) - -// fakeImageRegistry serves manifests and blobs of a small multi-arch image, -// a single-arch image and a moved tag. It fails with 503 once offline. -type fakeImageRegistry struct { - server *httptest.Server - offline atomic.Bool - // mu guards manifests against a test changing it while blobs are served. - mu sync.RWMutex - manifests map[string]fakeManifest - blobs map[string]string - blobGets atomic.Int32 - // tagGets counts GET requests for manifests by tag. - tagGets atomic.Int32 - // truncateManifests makes manifest GETs end after a few bytes. - truncateManifests atomic.Bool - // onBlob, if set, runs before a blob is served. - onBlob func(path string) - // sha512Digests makes manifests requested by a sha512 digest name that - // digest in Docker-Content-Digest, as registries do. - sha512Digests bool -} - -type fakeImage struct { - indexDigest string - amd64Digest string - arm64Digest string - amd64Layer string - arm64Layer string - singleDigest string - singleLayer string - // attestationLayers holds the amd64 and arm64 provenance statements. - attestationLayers []string -} - -func newFakeImageRegistry(t *testing.T) (*fakeImageRegistry, fakeImage) { - t.Helper() - const repository = "team/app" - r := &fakeImageRegistry{manifests: make(map[string]fakeManifest), blobs: make(map[string]string)} - var image fakeImage - - addBlob := func(content string) string { - digest := sha256Digest([]byte(content)) - r.blobs["/v2/"+repository+"/blobs/"+digest] = content - return digest - } - addManifest := func(contentType string, doc any, tags ...string) string { - body, err := json.Marshal(doc) - if err != nil { - t.Fatal(err) - } - digest := sha256Digest(body) - for _, reference := range append(tags, digest) { - r.manifests["/v2/"+repository+"/manifests/"+reference] = fakeManifest{contentType: contentType, body: string(body)} - } - return digest - } - imageManifest := func(platform string) (string, string) { - config := addBlob(`{"architecture":"` + platform + `"}`) - layer := addBlob("layer for " + platform) - digest := addManifest(ociManifestMediaType, map[string]any{ - "schemaVersion": 2, "mediaType": ociManifestMediaType, - "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": config}, - "layers": []map[string]any{ - {"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": layer}, - {"mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", "digest": sha256Digest([]byte("foreign")), "urls": []string{"https://example.invalid/layer"}}, - }, - }) - return digest, layer - } - - image.amd64Digest, image.amd64Layer = imageManifest("amd64") - image.arm64Digest, image.arm64Layer = imageManifest("arm64") - attestation := func(platformDigest, platform string) map[string]any { - statement := addBlob("provenance for " + platform) - digest := addManifest(ociManifestMediaType, map[string]any{ - "schemaVersion": 2, "mediaType": ociManifestMediaType, - "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": addBlob("{}")}, - "layers": []map[string]any{{"mediaType": "application/vnd.in-toto+json", "digest": statement}}, - }) - image.attestationLayers = append(image.attestationLayers, statement) - return map[string]any{ - "mediaType": ociManifestMediaType, "digest": digest, - "platform": map[string]string{"os": "unknown", "architecture": "unknown"}, - "annotations": map[string]string{ - dockerReferenceTypeAnnotation: attestationManifestReference, - dockerReferenceDigestAnnotation: platformDigest, - }, - } - } - image.indexDigest = addManifest(ociImageIndexMediaType, map[string]any{ - "schemaVersion": 2, "mediaType": ociImageIndexMediaType, - "manifests": []map[string]any{ - {"mediaType": ociManifestMediaType, "digest": image.amd64Digest, "platform": map[string]string{"os": "linux", "architecture": "amd64"}}, - {"mediaType": ociManifestMediaType, "digest": image.arm64Digest, "platform": map[string]string{"os": "linux", "architecture": "arm64", "variant": "v8"}}, - attestation(image.amd64Digest, "amd64"), - attestation(image.arm64Digest, "arm64"), - }, - }, "multi") - - image.singleDigest, image.singleLayer = imageManifest("single") - r.manifests["/v2/"+repository+"/manifests/single"] = r.manifests["/v2/"+repository+"/manifests/"+image.singleDigest] - - r.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if r.offline.Load() { - http.Error(w, "offline", http.StatusServiceUnavailable) - return - } - r.mu.RLock() - doc, ok := r.manifests[req.URL.Path] - r.mu.RUnlock() - if ok { - w.Header().Set("Content-Type", doc.contentType) - w.Header().Set("Docker-Content-Digest", sha256Digest([]byte(doc.body))) - if _, ref, _ := strings.Cut(req.URL.Path, "/manifests/"); r.sha512Digests && strings.HasPrefix(ref, "sha512:") { - w.Header().Set("Docker-Content-Digest", ref) - } - if req.Method == http.MethodGet && !strings.Contains(req.URL.Path, "/manifests/sha256:") { - r.tagGets.Add(1) - } - if req.Method == http.MethodGet && r.truncateManifests.Load() { - w.Header().Set("Content-Length", strconv.Itoa(len(doc.body))) - _, _ = io.WriteString(w, doc.body[:4]) - return - } - _, _ = io.WriteString(w, doc.body) - return - } - if blob, ok := r.blobs[req.URL.Path]; ok { - if r.onBlob != nil { - r.onBlob(req.URL.Path) - } - r.blobGets.Add(1) - _, _ = io.WriteString(w, blob) - return - } - http.NotFound(w, req) - })) - t.Cleanup(r.server.Close) - return r, image -} - -func newMirrorTestProxy(t *testing.T, registry *fakeImageRegistry) *Proxy { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = registry.server.Client() - proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(registry.server.Client()), fetch.WithMaxRetries(0)) - proxy.MetadataTTL = 0 - return proxy -} - -func pull(t *testing.T, h *ContainerHandler, target, accept string) *httptest.ResponseRecorder { - t.Helper() - request := httptest.NewRequest(http.MethodGet, target, nil) - if accept != "" { - request.Header.Set("Accept", accept) - } - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, request) - return response -} - -func TestMirrorImageThenPullOffline(t *testing.T) { - registry, image := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - mirrorHandler := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - - result, err := mirrorHandler.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"multi", "single"}}, nil) - if err != nil { - t.Fatalf("MirrorImage: %v", err) - } - if len(result.Warnings) == 0 || !strings.Contains(strings.Join(result.Warnings, "\n"), "non-distributable") { - t.Errorf("warnings = %v, want the skipped foreign layer", result.Warnings) - } - registry.offline.Store(true) - - // A fresh handler instance, as after a restart, serves the pull. - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - for _, accept := range []string{dockerManifestAccept, containerdManifestAccept} { - for _, reference := range []string{"multi", image.indexDigest, image.amd64Digest, image.arm64Digest, "single", image.singleDigest} { - if response := pull(t, h, "/team/app/manifests/"+reference, accept); response.Code != http.StatusOK { - t.Errorf("offline manifest %s with %.30q: status %d: %s", reference, accept, response.Code, response.Body.String()) - } - } - } - for _, layer := range append([]string{image.amd64Layer, image.arm64Layer, image.singleLayer}, image.attestationLayers...) { - if response := pull(t, h, "/team/app/blobs/"+layer, ""); response.Code != http.StatusOK { - t.Errorf("offline blob %s: status %d", layer, response.Code) - } - } - - registry.offline.Store(false) - gets := registry.blobGets.Load() - again, err := mirrorHandler.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"multi"}}, nil) - if err != nil { - t.Fatalf("second MirrorImage: %v", err) - } - if again.Fetched != 0 || registry.blobGets.Load() != gets { - t.Errorf("second run fetched %d items and %d blobs, want nothing new", again.Fetched, registry.blobGets.Load()-gets) - } -} - -func TestMirrorImagePlatformFilter(t *testing.T) { - registry, image := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - - platform, err := ParsePlatform("linux/arm64") - if err != nil { - t.Fatal(err) - } - if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"multi"}}, []Platform{platform}); err != nil { - t.Fatalf("MirrorImage: %v", err) - } - registry.offline.Store(true) - if response := pull(t, h, "/team/app/blobs/"+image.arm64Layer, ""); response.Code != http.StatusOK { - t.Errorf("arm64 layer status = %d, want 200", response.Code) - } - if response := pull(t, h, "/team/app/blobs/"+image.attestationLayers[1], ""); response.Code != http.StatusOK { - t.Errorf("arm64 attestation status = %d, want 200", response.Code) - } - for _, layer := range []string{image.amd64Layer, image.attestationLayers[0]} { - if response := pull(t, h, "/team/app/blobs/"+layer, ""); response.Code == http.StatusOK { - t.Errorf("amd64 blob %s was mirrored despite the linux/arm64 filter", layer) - } - } - - registry.offline.Store(false) - riscv, _ := ParsePlatform("linux/riscv64") - if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"multi"}}, []Platform{riscv}); err == nil { - t.Error("a filter matching no index entry did not fail") - } -} - -func TestMirrorImageMovedTagMirrorsBothTargets(t *testing.T) { - registry, image := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - - result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{image.singleDigest, "multi"}}, nil) - if err != nil { - t.Fatalf("MirrorImage: %v", err) - } - if !strings.Contains(strings.Join(result.Warnings, "\n"), "now points to") { - t.Errorf("warnings = %v, want the moved-tag warning", result.Warnings) - } - registry.offline.Store(true) - for _, layer := range []string{image.singleLayer, image.amd64Layer} { - if response := pull(t, h, "/team/app/blobs/"+layer, ""); response.Code != http.StatusOK { - t.Errorf("blob %s status = %d, want 200", layer, response.Code) - } - } -} - -func TestMirrorImageNamespaceRouteMatchesContainerdPull(t *testing.T) { - registry, image := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - named := map[string]string{"internal": registry.server.URL} - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", dockerHubRegistry, named) - host := strings.TrimPrefix(registry.server.URL, "http://") - - if _, err := h.MirrorImage(context.Background(), ImageRef{Registry: host, Name: "team/app", References: []string{"single"}}, nil); err != nil { - t.Fatalf("MirrorImage: %v", err) - } - registry.offline.Store(true) - ns := "?" + url.Values{namespaceQueryParam: {host}}.Encode() - if response := pull(t, h, "/team/app/manifests/single"+ns, containerdManifestAccept); response.Code != http.StatusOK { - t.Errorf("containerd ns manifest status = %d: %s", response.Code, response.Body.String()) - } - if response := pull(t, h, "/team/app/blobs/"+image.singleLayer+ns, ""); response.Code != http.StatusOK { - t.Errorf("containerd ns blob status = %d", response.Code) - } - if response := pull(t, h, "/upstream/internal/team/app/blobs/"+image.singleLayer, ""); response.Code != http.StatusOK { - t.Errorf("prefix-route blob status = %d", response.Code) - } - - if _, err := h.MirrorImage(context.Background(), ImageRef{Registry: "unknown.example", Name: "team/app", References: []string{"single"}}, nil); err == nil { - t.Error("an unconfigured registry host was accepted") - } -} - -func TestNormalizeImageRef(t *testing.T) { - tests := []struct { - name string - ref ImageRef - defaultRT string - wantName string - wantErr bool - }{ - {name: "docker hub default adds library", ref: ImageRef{Name: "alpine", References: []string{"3.20"}}, defaultRT: dockerHubRegistry, wantName: "library/alpine"}, - {name: "docker hub ns adds library", ref: ImageRef{Registry: "hub.docker.com", Name: "alpine", References: []string{"3.20"}}, defaultRT: "https://registry.example", wantName: "library/alpine"}, - {name: "other default keeps name", ref: ImageRef{Name: "alpine", References: []string{"3.20"}}, defaultRT: "https://registry.example", wantName: "alpine"}, - {name: "path traversal", ref: ImageRef{Name: "../../x", References: []string{"1"}}, defaultRT: dockerHubRegistry, wantErr: true}, - {name: "query in tag", ref: ImageRef{Name: "library/alpine", References: []string{"3.20?x#y"}}, defaultRT: dockerHubRegistry, wantErr: true}, - {name: "short digest", ref: ImageRef{Name: "library/alpine", References: []string{"sha256:abcd"}}, defaultRT: dockerHubRegistry, wantErr: true}, - {name: "no reference", ref: ImageRef{Name: "library/alpine"}, defaultRT: dockerHubRegistry, wantErr: true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := normalizeImageRef(tt.ref, tt.defaultRT) - if (err != nil) != tt.wantErr { - t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) - } - if !tt.wantErr && got.Name != tt.wantName { - t.Errorf("name = %q, want %q", got.Name, tt.wantName) - } - }) - } -} - -func TestMirrorImageKeepsPreviousTagWhenMovedTagIsIncomplete(t *testing.T) { - registry, image := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - ref := ImageRef{Name: "team/app", References: []string{"single"}} - - if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { - t.Fatalf("first MirrorImage: %v", err) - } - previous := registry.manifests["/v2/team/app/manifests/single"] - - // Move the tag to an image whose layer the registry cannot serve. - broken, err := json.Marshal(map[string]any{ - "schemaVersion": 2, "mediaType": ociManifestMediaType, - "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": image.singleLayer}, - "layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": sha256Digest([]byte("missing layer"))}}, - }) - if err != nil { - t.Fatal(err) - } - brokenDoc := fakeManifest{contentType: ociManifestMediaType, body: string(broken)} - registry.manifests["/v2/team/app/manifests/single"] = brokenDoc - registry.manifests["/v2/team/app/manifests/"+sha256Digest(broken)] = brokenDoc - - missing := sha256Digest([]byte("missing layer")) - if _, err := h.MirrorImage(context.Background(), ref, nil); err == nil || !strings.Contains(err.Error(), missing) { - t.Fatalf("mirroring the incomplete image: err = %v, want a failure naming layer %s", err, missing) - } - registry.offline.Store(true) - - response := pull(t, h, "/team/app/manifests/single", dockerManifestAccept) - if response.Code != http.StatusOK { - t.Fatalf("offline tag status = %d: %s", response.Code, response.Body.String()) - } - if response.Body.String() != previous.body { - t.Errorf("offline tag points to %s, want the previously mirrored image", sha256Digest(response.Body.Bytes())) - } -} - -func TestMirrorImagePlatformFilterAcrossNestedIndex(t *testing.T) { - registry, image := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - - nested := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` + - `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"` + image.arm64Digest + `","platform":{"os":"linux","architecture":"arm64"}}]}` - nestedDigest := sha256Digest([]byte(nested)) - top := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` + - `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"` + image.amd64Digest + `","platform":{"os":"linux","architecture":"amd64"}},` + - `{"mediaType":"application/vnd.oci.image.index.v1+json","digest":"` + nestedDigest + `"}]}` - for _, reference := range []string{"nested", nestedDigest} { - registry.manifests["/v2/team/app/manifests/"+reference] = fakeManifest{contentType: ociImageIndexMediaType, body: nested} - } - for _, reference := range []string{"top", sha256Digest([]byte(top))} { - registry.manifests["/v2/team/app/manifests/"+reference] = fakeManifest{contentType: ociImageIndexMediaType, body: top} - } - - amd64, _ := ParsePlatform("linux/amd64") - if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"top"}}, []Platform{amd64}); err != nil { - t.Fatalf("MirrorImage: %v (a nested index without amd64 must not fail the image)", err) - } - riscv, _ := ParsePlatform("linux/riscv64") - if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"top"}}, []Platform{riscv}); err == nil { - t.Error("a filter matching nothing in the whole image did not fail") - } -} - -func TestMirrorImageRevalidatesUnchangedTagWithHead(t *testing.T) { - registry, _ := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - ref := ImageRef{Name: "team/app", References: []string{"multi"}} - - if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { - t.Fatalf("first MirrorImage: %v", err) - } - gets := registry.tagGets.Load() - if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { - t.Fatalf("second MirrorImage: %v", err) - } - if extra := registry.tagGets.Load() - gets; extra != 0 { - t.Errorf("second run sent %d GETs for the unchanged tag, want a HEAD only", extra) - } -} - -func TestMirrorImageFallsBackWhenTagBodyIsCutOff(t *testing.T) { - registry, image := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - ref := ImageRef{Name: "team/app", References: []string{"single"}} - - if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { - t.Fatalf("first MirrorImage: %v", err) - } - // Point the tag elsewhere so the HEAD check sends a GET, then cut that - // GET's body short. - registry.manifests["/v2/team/app/manifests/single"] = registry.manifests["/v2/team/app/manifests/"+image.amd64Digest] - registry.truncateManifests.Store(true) - - result, err := h.MirrorImage(context.Background(), ref, nil) - if err != nil { - t.Fatalf("MirrorImage with a cut-off body: %v, want the cached tag to be used", err) - } - if !strings.Contains(strings.Join(result.Warnings, "\n"), "stale cache") { - t.Errorf("warnings = %v, want the stale-cache warning", result.Warnings) - } -} - -func TestMirrorImageKeepsTagMovedByPullDuringWalk(t *testing.T) { - registry, image := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - - // While the mirror fetches the first layer of "single", the tag moves - // upstream and a client pull caches the new target. - moved := registry.manifests["/v2/team/app/manifests/"+image.amd64Digest] - var once atomic.Bool - registry.onBlob = func(string) { - if once.CompareAndSwap(false, true) { - registry.mu.Lock() - registry.manifests["/v2/team/app/manifests/single"] = moved - registry.mu.Unlock() - if response := pull(t, h, "/team/app/manifests/single", dockerManifestAccept); response.Code != http.StatusOK { - t.Errorf("client pull during the walk: status %d", response.Code) - } - } - } - result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"single"}}, nil) - if err != nil { - t.Fatalf("MirrorImage: %v", err) - } - if !strings.Contains(strings.Join(result.Warnings, "\n"), "moved") { - t.Errorf("warnings = %v, want the moved-tag warning", result.Warnings) - } - registry.onBlob = nil - registry.offline.Store(true) - if response := pull(t, h, "/team/app/manifests/single", dockerManifestAccept); response.Body.String() != moved.body { - t.Errorf("offline tag points to %s, want the newer target %s", sha256Digest(response.Body.Bytes()), image.amd64Digest) - } -} - -func TestMirrorImageCountsNoCacheHits(t *testing.T) { - registry, _ := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - ref := ImageRef{Name: "team/app", References: []string{"multi"}} - - if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { - t.Fatalf("first MirrorImage: %v", err) - } - var hits int - var lastAccess sql.NullString - if err := proxy.DB.QueryRow("SELECT COALESCE(SUM(hit_count), 0), MAX(last_accessed_at) FROM artifacts").Scan(&hits, &lastAccess); err != nil { - t.Fatal(err) - } - if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { - t.Fatalf("second MirrorImage: %v", err) - } - var hitsAfter int - var lastAccessAfter sql.NullString - if err := proxy.DB.QueryRow("SELECT COALESCE(SUM(hit_count), 0), MAX(last_accessed_at) FROM artifacts").Scan(&hitsAfter, &lastAccessAfter); err != nil { - t.Fatal(err) - } - if hitsAfter != hits || lastAccessAfter != lastAccess { - t.Errorf("second run changed hits %d -> %d and last access %v -> %v, want no change", hits, hitsAfter, lastAccess, lastAccessAfter) - } -} - -// A scheduled image that no client pulls keeps its blobs: a mirror run that -// finds them cached holds them back from retention, though not from LRU -// eviction, which TestMirrorImageCountsNoCacheHits covers. -func TestMirrorImageKeepsCachedBlobsFromRetention(t *testing.T) { - registry, _ := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - ref := ImageRef{Name: "team/app", References: []string{"multi"}} - - if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { - t.Fatalf("first MirrorImage: %v", err) - } - old := time.Now().Add(-30 * 24 * time.Hour).UTC() - if _, err := proxy.DB.Exec(proxy.DB.Rebind("UPDATE artifacts SET fetched_at = ?, last_accessed_at = ?"), old, old); err != nil { - t.Fatal(err) - } - cutoff := time.Now().Add(-14 * 24 * time.Hour) - maxID, err := proxy.DB.MaxArtifactID() - if err != nil { - t.Fatal(err) - } - if candidates, err := proxy.DB.GetRetentionCandidates(0, maxID, cutoff); err != nil || len(candidates) == 0 { - t.Fatalf("before the second run: %d candidates, err %v; want the aged blobs", len(candidates), err) - } - - if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { - t.Fatalf("second MirrorImage: %v", err) - } - candidates, err := proxy.DB.GetRetentionCandidates(0, maxID, cutoff) - if err != nil { - t.Fatal(err) - } - if len(candidates) != 0 { - t.Errorf("after the second run %d blobs are retention candidates, want none", len(candidates)) - } -} - -func TestMirrorImageRefetchesBlobMissingFromStorage(t *testing.T) { - registry, image := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - ref := ImageRef{Name: "team/app", References: []string{"multi"}} - - if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { - t.Fatalf("first MirrorImage: %v", err) - } - var path string - if err := proxy.DB.QueryRow("SELECT storage_path FROM artifacts WHERE filename = ?", image.amd64Layer).Scan(&path); err != nil { - t.Fatal(err) - } - if err := proxy.Storage.Delete(context.Background(), path); err != nil { - t.Fatal(err) - } - gets := registry.blobGets.Load() - if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { - t.Fatalf("second MirrorImage: %v", err) - } - if got := registry.blobGets.Load() - gets; got != 1 { - t.Errorf("second run fetched %d blobs, want the one missing from storage", got) - } -} - -func TestMirrorImageFetchesBlobsConcurrently(t *testing.T) { - registry, _ := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - - // "single" has a config and a layer blob. Each request waits until the - // other is in flight too, or gives up after a while. - var inFlight atomic.Int32 - var concurrent atomic.Bool - both := make(chan struct{}) - var once sync.Once - registry.onBlob = func(string) { - defer inFlight.Add(-1) - if inFlight.Add(1) >= 2 { - concurrent.Store(true) - once.Do(func() { close(both) }) - } - select { - case <-both: - case <-time.After(2 * time.Second): - } - } - result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"single"}}, nil) - if err != nil { - t.Fatalf("MirrorImage: %v", err) - } - if !concurrent.Load() { - t.Error("blobs were fetched one after another, want them fetched concurrently") - } - if result.Fetched != 3 { - t.Errorf("fetched = %d, want the manifest and its two blobs", result.Fetched) - } -} - -// TestMirrorImageBlobFailureKeepsOtherFetches checks that a blob failing -// doesn't cancel the blobs already being fetched: their fetch may be shared -// with a client pull, which would fail with it, and the next run would have -// to fetch them again. -func TestMirrorImageBlobFailureKeepsOtherFetches(t *testing.T) { - registry, image := newFakeImageRegistry(t) - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - - // The config blob is gone upstream; the layer is still being sent when - // its 404 arrives. - for path := range registry.blobs { - if !strings.HasSuffix(path, image.singleLayer) { - delete(registry.blobs, path) - } - } - registry.onBlob = func(string) { time.Sleep(300 * time.Millisecond) } - - if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"single"}}, nil); err == nil { - t.Fatal("MirrorImage succeeded with a blob missing upstream") - } - registry.offline.Store(true) - if response := pull(t, h, "/team/app/blobs/"+image.singleLayer, ""); response.Code != http.StatusOK { - t.Errorf("layer fetched alongside the failing blob: status %d, want it cached", response.Code) - } -} - -func TestMirrorImageAcceptsSHA512Blobs(t *testing.T) { - registry, image := newFakeImageRegistry(t) - const layer = "sha512 layer" - sum := sha512.Sum512([]byte(layer)) - layerDigest := "sha512:" + hex.EncodeToString(sum[:]) - body, err := json.Marshal(map[string]any{ - "schemaVersion": 2, "mediaType": ociManifestMediaType, - "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": image.singleLayer}, - "layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": layerDigest}}, - }) - if err != nil { - t.Fatal(err) - } - registry.mu.Lock() - registry.manifests["/v2/team/app/manifests/sha512"] = fakeManifest{contentType: ociManifestMediaType, body: string(body)} - registry.blobs["/v2/team/app/blobs/"+layerDigest] = layer - registry.mu.Unlock() - - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"sha512"}}, nil); err != nil { - t.Fatalf("MirrorImage: %v", err) - } - cached, err := proxy.GetCachedArtifact(t.Context(), "oci", "team/app", layerDigest, layerDigest) - if err != nil || cached == nil { - t.Fatalf("sha512 layer not cached: %v, %v", cached, err) - } - if cached.Reader != nil { - _ = cached.Reader.Close() - } -} - -// TestMirrorImageSHA512ReferenceMatchesTag checks that mirroring a tag -// pinned by a sha512 digest compares it with the tag's manifest by sha512, -// so an unmoved tag is neither reported as moved nor counted twice. -func TestMirrorImageSHA512ReferenceMatchesTag(t *testing.T) { - mirror := func(t *testing.T, sha512Ref bool) ImageResult { - registry, image := newFakeImageRegistry(t) - reference := image.singleDigest - if sha512Ref { - body := registry.manifests["/v2/team/app/manifests/single"].body - sum := sha512.Sum512([]byte(body)) - reference = "sha512:" + hex.EncodeToString(sum[:]) - registry.manifests["/v2/team/app/manifests/"+reference] = registry.manifests["/v2/team/app/manifests/single"] - registry.sha512Digests = true - } - proxy := newMirrorTestProxy(t, registry) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) - result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{reference, "single"}}, nil) - if err != nil { - t.Fatalf("MirrorImage: %v", err) - } - return result - } - want := mirror(t, false) - got := mirror(t, true) - if !slices.Equal(got.Warnings, want.Warnings) { - t.Errorf("warnings = %v, want %v as for a sha256 reference", got.Warnings, want.Warnings) - } - if got.Fetched != want.Fetched { - t.Errorf("fetched = %d, want %d as for a sha256 reference", got.Fetched, want.Fetched) - } -} diff --git a/internal/handler/container_ns_test.go b/internal/handler/container_ns_test.go deleted file mode 100644 index e4a1e12..0000000 --- a/internal/handler/container_ns_test.go +++ /dev/null @@ -1,801 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "log/slog" - "net" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "sync" - "testing" - "time" - - "github.com/git-pkgs/registries/fetch" -) - -const ( - nsTestBlob = "layer bytes" - nsTestProxyURL = "http://proxy.example.test" -) - -// nsTestRegistry is a fake OCI registry serving one repository below an -// optional path prefix. It records every request URI it receives. -type nsTestRegistry struct { - *httptest.Server - repository string - - mu sync.Mutex - requests []string -} - -func newNSTestRegistry(t *testing.T, repository, pathPrefix string) *nsTestRegistry { - t.Helper() - registry := &nsTestRegistry{repository: repository} - manifest := nsTestManifest() - blobDigest := "sha256:" + sha256Hex(nsTestBlob) - manifestDigest := "sha256:" + sha256Hex(manifest) - base := pathPrefix + "/v2/" + repository - registry.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - registry.mu.Lock() - registry.requests = append(registry.requests, r.URL.RequestURI()) - registry.mu.Unlock() - - switch r.URL.Path { - case base + "/manifests/latest", base + "/manifests/" + manifestDigest: - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Set("Docker-Content-Digest", manifestDigest) - _, _ = io.WriteString(w, manifest) - case base + "/blobs/" + blobDigest: - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = io.WriteString(w, nsTestBlob) - case base + "/tags/list": - w.Header().Set("Content-Type", "application/json") - if r.URL.Query().Get("last") == "" { - w.Header().Set("Link", `<`+base+`/tags/list?last=1.0&n=1>; rel="next"`) - _, _ = io.WriteString(w, `{"name":"`+repository+`","tags":["1.0"]}`) - return - } - _, _ = io.WriteString(w, `{"name":"`+repository+`","tags":["2.0"]}`) - default: - http.NotFound(w, r) - } - })) - t.Cleanup(registry.Close) - return registry -} - -func nsTestManifest() string { - return `{"schemaVersion":2,"layers":[{"digest":"sha256:` + sha256Hex(nsTestBlob) + `"}]}` -} - -func (r *nsTestRegistry) host() string { - return strings.TrimPrefix(r.URL, "http://") -} - -func (r *nsTestRegistry) requestCount() int { - r.mu.Lock() - defer r.mu.Unlock() - return len(r.requests) -} - -func (r *nsTestRegistry) lastRequest() string { - r.mu.Lock() - defer r.mu.Unlock() - if len(r.requests) == 0 { - return "" - } - return r.requests[len(r.requests)-1] -} - -// newNSTestHandler builds a container handler the way the server does, with a -// real fetcher so blob downloads reach the fake registries. Warnings logged -// while building the handler are written to the returned buffer. -func newNSTestHandler(t *testing.T, defaultURL string, named map[string]string) (http.Handler, *ContainerHandler, *bytes.Buffer) { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - logs := &bytes.Buffer{} - proxy.Logger = slog.New(slog.NewTextHandler(logs, nil)) - client := &http.Client{} - proxy.HTTPClient = client - proxy.MetadataTTL = time.Hour - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewContainerHandlerWithRegistry(proxy, nsTestProxyURL, defaultURL, named) - return http.StripPrefix("/v2", h.Routes()), h, logs -} - -func serveNS(routes http.Handler, target string) *httptest.ResponseRecorder { - response := httptest.NewRecorder() - routes.ServeHTTP(response, httptest.NewRequest(http.MethodGet, target, nil)) - return response -} - -func assertNameUnknown(t *testing.T, response *httptest.ResponseRecorder) { - t.Helper() - if response.Code != http.StatusNotFound { - t.Fatalf("status = %d, want 404: %s", response.Code, response.Body.String()) - } - if !strings.Contains(response.Body.String(), `"NAME_UNKNOWN"`) { - t.Errorf("body = %s, want NAME_UNKNOWN error", response.Body.String()) - } -} - -func TestContainerHandler_NamespaceSelectsDefaultRegistry(t *testing.T) { - registry := newNSTestRegistry(t, "library/nginx", "") - // The default registry is a custom oci_default, so its host is only - // resolvable when the index is built from the final registry URL. - routes, _, _ := newNSTestHandler(t, registry.URL, nil) - - for _, namespace := range []string{"docker.io", "index.docker.io", "registry-1.docker.io", registry.host()} { - t.Run(namespace, func(t *testing.T) { - response := serveNS(routes, "/v2/library/nginx/manifests/latest?ns="+namespace) - if response.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", response.Code, response.Body.String()) - } - if got, want := registry.lastRequest(), "/v2/library/nginx/manifests/latest"; got != want { - t.Errorf("upstream request = %q, want %q", got, want) - } - }) - } -} - -func TestContainerHandler_NamespaceSelectsNamedRegistry(t *testing.T) { - fallback := newNSTestRegistry(t, "owner/app", "") - named := newNSTestRegistry(t, "owner/app", "") - routes, _, _ := newNSTestHandler(t, fallback.URL, map[string]string{"ghcr": named.URL}) - digest := "sha256:" + sha256Hex(nsTestBlob) - - manifest := serveNS(routes, "/v2/owner/app/manifests/latest?ns="+named.host()) - if manifest.Code != http.StatusOK { - t.Fatalf("manifest status = %d, want 200: %s", manifest.Code, manifest.Body.String()) - } - blob := serveNS(routes, "/v2/owner/app/blobs/"+digest+"?ns="+named.host()) - if blob.Code != http.StatusOK { - t.Fatalf("blob status = %d, want 200: %s", blob.Code, blob.Body.String()) - } - if got := blob.Body.String(); got != nsTestBlob { - t.Errorf("blob body = %q, want %q", got, nsTestBlob) - } - if got := named.requestCount(); got != 2 { - t.Errorf("named registry requests = %d, want 2", got) - } - if got := fallback.requestCount(); got != 0 { - t.Errorf("default registry requests = %d, want 0", got) - } -} - -func TestContainerHandler_NamespaceRejectsUnresolvableRequests(t *testing.T) { - registry := newNSTestRegistry(t, "owner/app", "") - other := newNSTestRegistry(t, "owner/app", "") - routes, _, _ := newNSTestHandler(t, registry.URL, map[string]string{"ghcr": registry.URL, "quay": other.URL}) - digest := "sha256:" + sha256Hex(nsTestBlob) - - tests := map[string]string{ - "unknown host manifest": "/v2/owner/app/manifests/latest?ns=quay.io", - "unknown host blob": "/v2/owner/app/blobs/" + digest + "?ns=quay.io", - "unknown host tags": "/v2/owner/app/tags/list?ns=quay.io", - "multiple ns values": "/v2/owner/app/manifests/latest?ns=docker.io&ns=" + registry.host(), - "prefix route with default host": "/v2/upstream/quay/owner/app/manifests/latest?ns=" + registry.host(), - "prefix route with other upstream": "/v2/upstream/ghcr/owner/app/blobs/" + digest + "?ns=" + other.host(), - "prefix route with unknown upstream": "/v2/upstream/nope/owner/app/manifests/latest?ns=" + registry.host(), - "prefix route without repository": "/v2/upstream/ghcr/manifests/latest?ns=" + registry.host(), - } - for name, target := range tests { - t.Run(name, func(t *testing.T) { - assertNameUnknown(t, serveNS(routes, target)) - }) - } - if got := registry.requestCount() + other.requestCount(); got != 0 { - t.Errorf("upstream requests = %d, want 0", got) - } -} - -func TestContainerHandler_NamespaceAcceptsPrefixRouteForOwnHost(t *testing.T) { - // Per-registry containerd mirrors with override_path address the - // upstream/{name}/ prefix and still send ns. They must keep working and - // share the prefix route's cache entries. - registry := newNSTestRegistry(t, "owner/app", "") - routes, _, _ := newNSTestHandler(t, "", map[string]string{"ghcr": registry.URL}) - digest := "sha256:" + sha256Hex(nsTestBlob) - - for _, target := range []string{ - "/v2/upstream/ghcr/owner/app/manifests/latest?ns=" + registry.host(), - "/v2/upstream/ghcr/owner/app/blobs/" + digest + "?ns=" + registry.host(), - } { - if response := serveNS(routes, target); response.Code != http.StatusOK { - t.Fatalf("%s status = %d: %s", target, response.Code, response.Body.String()) - } - } - if got, want := registry.lastRequest(), "/v2/owner/app/blobs/"+digest; got != want { - t.Errorf("upstream request = %q, want %q", got, want) - } - warmed := registry.requestCount() - - for _, target := range []string{ - "/v2/upstream/ghcr/owner/app/manifests/latest", - "/v2/upstream/ghcr/owner/app/blobs/" + digest, - } { - if response := serveNS(routes, target); response.Code != http.StatusOK { - t.Fatalf("%s status = %d: %s", target, response.Code, response.Body.String()) - } - } - if got := registry.requestCount(); got != warmed { - t.Errorf("upstream requests after prefix pulls without ns = %d, want %d (cache hits)", got, warmed) - } - - tags := serveNS(routes, "/v2/upstream/ghcr/owner/app/tags/list?n=1&ns="+registry.host()) - if tags.Code != http.StatusOK { - t.Fatalf("tags status = %d: %s", tags.Code, tags.Body.String()) - } - wantLink := `<` + nsTestProxyURL + `/v2/upstream/ghcr/owner/app/tags/list?last=1.0&n=1&ns=` + url.QueryEscape(registry.host()) + `>; rel="next"` - if got := tags.Header().Get("Link"); got != wantLink { - t.Errorf("Link = %q, want %q", got, wantLink) - } - if got, want := registry.lastRequest(), "/v2/owner/app/tags/list?n=1"; got != want { - t.Errorf("upstream tags request = %q, want %q (ns must not be forwarded)", got, want) - } -} - -func TestContainerHandler_NamespacePrefixRouteAcceptsDockerHubAliases(t *testing.T) { - // upstream.oci.hub points at Docker Hub and a docker.io hosts.toml mirror - // with override_path addresses /v2/upstream/hub, so containerd sends - // ns=docker.io. The dialer stands in for registry-1.docker.io. - hub := newNSTestRegistry(t, "library/nginx", "") - other := newNSTestRegistry(t, "library/nginx", "") - routes, h, _ := newNSTestHandler(t, "", map[string]string{ - "hub": "http://registry-1.docker.io", - "quay": other.URL, - }) - transport := &http.Transport{DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - if addr != "registry-1.docker.io:80" { - return nil, fmt.Errorf("unexpected dial to %s", addr) - } - return (&net.Dialer{}).DialContext(ctx, network, hub.Listener.Addr().String()) - }} - t.Cleanup(transport.CloseIdleConnections) - h.proxy.HTTPClient = &http.Client{Transport: transport} - - for _, query := range []string{"?ns=docker.io", "?ns=index.docker.io", "?ns=registry-1.docker.io", "?ns=docker.io:443", ""} { - response := serveNS(routes, "/v2/upstream/hub/library/nginx/manifests/latest"+query) - if response.Code != http.StatusOK { - t.Fatalf("%q status = %d: %s", query, response.Code, response.Body.String()) - } - } - if got := hub.requestCount(); got != 1 { - t.Errorf("hub requests = %d, want 1 (the rest are cache hits)", got) - } - - // A known host of another route still contradicts the prefix. - assertNameUnknown(t, serveNS(routes, "/v2/upstream/hub/library/nginx/manifests/latest?ns="+other.host())) - if got := other.requestCount(); got != 0 { - t.Errorf("other registry requests = %d, want 0", got) - } - - t.Run("Docker Hub mirror as named upstream", func(t *testing.T) { - // The upstream is a Docker Hub mirror on some other host (mirror.gcr.io, - // an Artifactory remote); nodes still pull docker.io/... through - // /v2/upstream/hub, so ns says docker.io. - mirror := newNSTestRegistry(t, "library/nginx", "") - routes, _, _ := newNSTestHandler(t, "", map[string]string{"hub": mirror.URL}) - - response := serveNS(routes, "/v2/upstream/hub/library/nginx/manifests/latest?ns=docker.io") - if response.Code != http.StatusOK { - t.Fatalf("status = %d: %s", response.Code, response.Body.String()) - } - if got, want := mirror.lastRequest(), "/v2/library/nginx/manifests/latest"; got != want { - t.Errorf("upstream request = %q, want %q", got, want) - } - }) -} - -func TestContainerHandler_NamespacePrefixRouteRejectsUnknownHosts(t *testing.T) { - // A _default wildcard mirror turns a pull of - // unconfigured.example/upstream/mirror/owner/app into exactly this - // request. Serving it would hand out owner/app from the mirror upstream - // instead of letting containerd fall back to unconfigured.example. - mirror := newNSTestRegistry(t, "owner/app", "") - routes, h, logs := newNSTestHandler(t, "", map[string]string{"mirror": mirror.URL}) - - for _, namespace := range []string{"unconfigured.example", "ghcr.io"} { - assertNameUnknown(t, serveNS(routes, "/v2/upstream/mirror/owner/app/manifests/latest?ns="+namespace)) - } - if got := mirror.requestCount(); got != 0 { - t.Errorf("upstream requests = %d, want 0", got) - } - // containerd swallows the 404, so the refusal has to show up in the log. - for _, want := range []string{"upstream.oci_mirrors", "upstream=mirror", "ns=ghcr.io"} { - if !strings.Contains(logs.String(), want) { - t.Errorf("logs missing %q:\n%s", want, logs.String()) - } - } - - // A node repeats the refused request for every pull; one line per - // upstream/ns pair is enough. An ns longer than any registry host is - // refused before it is logged or remembered. - assertNameUnknown(t, serveNS(routes, "/v2/upstream/mirror/owner/app/manifests/latest?ns=GHCR.io")) - long := strings.Repeat("a", 300) + ".example" - assertNameUnknown(t, serveNS(routes, "/v2/upstream/mirror/owner/app/manifests/latest?ns="+long)) - if got := strings.Count(logs.String(), "upstream.oci_mirrors"); got != 2 { - t.Errorf("refusal warnings = %d, want 2 (one per upstream/ns pair):\n%s", got, logs.String()) - } - if strings.Contains(logs.String(), "aaaa") { - t.Errorf("logs mention the overlong ns, want it refused silently:\n%s", logs.String()) - } - if got := len(h.refusals); got != 2 { - t.Errorf("remembered refusals = %d, want 2", got) - } -} - -func TestContainerHandler_NamespaceRefusalLogIsBounded(t *testing.T) { - mirror := newNSTestRegistry(t, "owner/app", "") - routes, h, logs := newNSTestHandler(t, "", map[string]string{"mirror": mirror.URL}) - h.refusals = make(map[string]struct{}, maxReportedNamespaceRefusals) - for i := range maxReportedNamespaceRefusals { - h.refusals[fmt.Sprintf("mirror\x00filler-%d.example", i)] = struct{}{} - } - - for _, namespace := range []string{"filler-0.example", "first.example", "second.example"} { - assertNameUnknown(t, serveNS(routes, "/v2/upstream/mirror/owner/app/manifests/latest?ns="+namespace)) - } - if got := strings.Count(logs.String(), "further refusals are not logged"); got != 1 { - t.Errorf("limit notices = %d, want 1:\n%s", got, logs.String()) - } - if strings.Contains(logs.String(), "refusing ns") { - t.Errorf("refusal logged past the limit:\n%s", logs.String()) - } - if got := len(h.refusals); got != maxReportedNamespaceRefusals { - t.Errorf("remembered refusals = %d, want %d", got, maxReportedNamespaceRefusals) - } -} - -func TestContainerHandler_NamespaceMirroredRegistries(t *testing.T) { - // The upstream mirrors ghcr.io, and upstream.oci_mirrors says so. Nodes - // keep pulling ghcr.io/..., either through a ghcr.io hosts.toml pointing - // at /v2/upstream/ghcr or through the _default wildcard mirror. - tests := map[string]string{ - "plain mirror host": "", - "artifactory remote": "/artifactory/api/docker/ghcr-remote", - } - for name, pathPrefix := range tests { - t.Run(name, func(t *testing.T) { - mirror := newNSTestRegistry(t, "owner/app", pathPrefix) - routes, h, _ := newNSTestHandler(t, "", map[string]string{"ghcr": mirror.URL + pathPrefix}) - h.SetMirroredRegistries(map[string][]string{"ghcr": {"ghcr.io"}}) - - for _, target := range []string{ - "/v2/upstream/ghcr/owner/app/manifests/latest?ns=ghcr.io", - "/v2/upstream/ghcr/owner/app/manifests/latest?ns=GHCR.io:443", - "/v2/owner/app/manifests/latest?ns=ghcr.io", - } { - before := mirror.requestCount() - response := serveNS(routes, target) - if response.Code != http.StatusOK { - t.Fatalf("GET %s status = %d: %s", target, response.Code, response.Body.String()) - } - // The manifest is cached after the first request, so only - // the first one may reach the upstream. - if before == 0 { - if got, want := mirror.lastRequest(), pathPrefix+"/v2/owner/app/manifests/latest"; got != want { - t.Errorf("upstream request = %q, want %q", got, want) - } - } - } - if got := mirror.requestCount(); got != 1 { - t.Errorf("upstream requests = %d, want 1 (all routes share the cache)", got) - } - - assertNameUnknown(t, serveNS(routes, "/v2/upstream/ghcr/owner/app/manifests/latest?ns=quay.io")) - }) - } -} - -func TestContainerHandler_NamespaceMirrorHostsRankBelowConfiguredURLs(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - logs := &bytes.Buffer{} - proxy.Logger = slog.New(slog.NewTextHandler(logs, nil)) - h := NewContainerHandlerWithRegistry(proxy, nsTestProxyURL, "", map[string]string{ - "ghcr": "https://ghcr.io", - "art": "https://artifactory.example/api/docker/ghcr-remote", - "other": "https://other.example", - }) - h.SetMirroredRegistries(map[string][]string{"art": {"ghcr.io"}, "missing": {"quay.io"}}) - - if got := h.namespaces["ghcr.io"]; got != "ghcr" { - t.Errorf("index[ghcr.io] = %q, want ghcr (configured URL wins over a mirror entry)", got) - } - if !h.namespaceNamesPrefixUpstream("ghcr.io", "upstream/art/owner/app") { - t.Error("ns ghcr.io refused on upstream/art/, want accepted as its mirrored registry") - } - if h.namespaceNamesPrefixUpstream("ghcr.io", "upstream/other/owner/app") { - t.Error("ns ghcr.io accepted on upstream/other/, want refused: only art lists it") - } - if _, ok := h.namespaces["quay.io"]; ok { - t.Error("index has quay.io from a mirror entry without upstream") - } - for _, want := range []string{"ghcr.io=ghcr", "upstream=missing"} { - if !strings.Contains(logs.String(), want) { - t.Errorf("logs missing %q:\n%s", want, logs.String()) - } - } -} - -func TestContainerHandler_NamespaceSharesCacheWithOtherRoutes(t *testing.T) { - digest := "sha256:" + sha256Hex(nsTestBlob) - manifestDigest := "sha256:" + sha256Hex(nsTestManifest()) - - t.Run("named registry", func(t *testing.T) { - registry := newNSTestRegistry(t, "owner/app", "") - routes, _, _ := newNSTestHandler(t, "", map[string]string{"ghcr": registry.URL}) - - for _, target := range []string{ - "/v2/upstream/ghcr/owner/app/blobs/" + digest, - "/v2/upstream/ghcr/owner/app/manifests/" + manifestDigest, - } { - if response := serveNS(routes, target); response.Code != http.StatusOK { - t.Fatalf("warm %s status = %d: %s", target, response.Code, response.Body.String()) - } - } - warmed := registry.requestCount() - for _, target := range []string{ - "/v2/owner/app/blobs/" + digest + "?ns=" + registry.host(), - "/v2/owner/app/manifests/" + manifestDigest + "?ns=" + registry.host(), - } { - if response := serveNS(routes, target); response.Code != http.StatusOK { - t.Fatalf("ns %s status = %d: %s", target, response.Code, response.Body.String()) - } - } - if got := registry.requestCount(); got != warmed { - t.Errorf("upstream requests after ns pulls = %d, want %d (cache hits)", got, warmed) - } - }) - - t.Run("default registry", func(t *testing.T) { - registry := newNSTestRegistry(t, "library/nginx", "") - routes, _, _ := newNSTestHandler(t, registry.URL, nil) - - for _, target := range []string{ - "/v2/library/nginx/blobs/" + digest + "?ns=docker.io", - "/v2/library/nginx/manifests/" + manifestDigest + "?ns=docker.io", - } { - if response := serveNS(routes, target); response.Code != http.StatusOK { - t.Fatalf("warm %s status = %d: %s", target, response.Code, response.Body.String()) - } - } - warmed := registry.requestCount() - for _, target := range []string{ - "/v2/library/nginx/blobs/" + digest, - "/v2/library/nginx/manifests/" + manifestDigest, - } { - if response := serveNS(routes, target); response.Code != http.StatusOK { - t.Fatalf("unprefixed %s status = %d: %s", target, response.Code, response.Body.String()) - } - } - if got := registry.requestCount(); got != warmed { - t.Errorf("upstream requests after unprefixed pulls = %d, want %d (cache hits)", got, warmed) - } - }) -} - -func TestContainerHandler_NamespaceTagsList(t *testing.T) { - registry := newNSTestRegistry(t, "owner/app", "") - routes, _, _ := newNSTestHandler(t, "", map[string]string{"ghcr": registry.URL}) - - // The prefix route fills the shared cache entry first. - prefixed := serveNS(routes, "/v2/upstream/ghcr/owner/app/tags/list?n=1") - if prefixed.Code != http.StatusOK { - t.Fatalf("prefixed status = %d: %s", prefixed.Code, prefixed.Body.String()) - } - const wantPrefixedLink = `<` + nsTestProxyURL + `/v2/upstream/ghcr/owner/app/tags/list?last=1.0&n=1>; rel="next"` - if got := prefixed.Header().Get("Link"); got != wantPrefixedLink { - t.Errorf("prefixed Link = %q, want %q", got, wantPrefixedLink) - } - - namespaced := serveNS(routes, "/v2/owner/app/tags/list?n=1&ns="+registry.host()) - if namespaced.Code != http.StatusOK { - t.Fatalf("ns status = %d: %s", namespaced.Code, namespaced.Body.String()) - } - if got := registry.requestCount(); got != 1 { - t.Errorf("upstream requests = %d, want 1 (ns request served from the shared cache)", got) - } - wantNSLink := `<` + nsTestProxyURL + `/v2/owner/app/tags/list?last=1.0&n=1&ns=` + url.QueryEscape(registry.host()) + `>; rel="next"` - link := namespaced.Header().Get("Link") - if link != wantNSLink { - t.Fatalf("ns Link = %q, want %q", link, wantNSLink) - } - - next := serveNS(routes, strings.TrimPrefix(strings.SplitN(link, ">", 2)[0], "<")) - if next.Code != http.StatusOK { - t.Fatalf("next page status = %d: %s", next.Code, next.Body.String()) - } - if got, want := next.Body.String(), `{"name":"owner/app","tags":["2.0"]}`; got != want { - t.Errorf("next page body = %q, want %q", got, want) - } - if got, want := registry.lastRequest(), "/v2/owner/app/tags/list?last=1.0&n=1"; got != want { - t.Errorf("upstream request = %q, want %q (ns must not be forwarded)", got, want) - } -} - -func TestContainerHandler_NamespaceDefaultBypassesRepositoryPrefixRoutes(t *testing.T) { - hub := newNSTestRegistry(t, "homebrew/core/jq", "") - brew := newNSTestRegistry(t, "homebrew/core/jq", "") - routes, h, _ := newNSTestHandler(t, hub.URL, nil) - RegisterHomebrewArtifacts(h, brew.URL) - - if response := serveNS(routes, "/v2/homebrew/core/jq/manifests/latest?ns=docker.io"); response.Code != http.StatusOK { - t.Fatalf("ns status = %d: %s", response.Code, response.Body.String()) - } - if hub.requestCount() != 1 || brew.requestCount() != 0 { - t.Errorf("ns request reached hub=%d brew=%d, want hub=1 brew=0", hub.requestCount(), brew.requestCount()) - } - - if response := serveNS(routes, "/v2/homebrew/core/jq/manifests/latest"); response.Code != http.StatusOK { - t.Fatalf("unprefixed status = %d: %s", response.Code, response.Body.String()) - } - if hub.requestCount() != 1 || brew.requestCount() != 1 { - t.Errorf("unprefixed request reached hub=%d brew=%d, want hub=1 brew=1", hub.requestCount(), brew.requestCount()) - } -} - -func TestContainerHandler_NamespaceMatchesConfiguredHosts(t *testing.T) { - // Each pair is a configured registry URL and an ns value containerd sends - // for an image reference on that registry. The scheme-default port may be - // spelled out or left out on either side; any other port must match. - matches := []struct { - configured string - namespace string - }{ - {"https://GHCR.io", "ghcr.io"}, - {"https://ghcr.io/", "GHCR.IO"}, - {"http://[fd00::1]:5000", "[fd00::1]:5000"}, - {"https://[fd00::1]", "[fd00::1]"}, - {"https://[fd00::1]", "[fd00::1]:443"}, - {"https://reg.example:80", "reg.example:80"}, - {"https://reg.example:443", "reg.example"}, - {"https://reg.example", "reg.example:443"}, - {"http://reg.example:80", "reg.example"}, - {"http://reg.example", "reg.example:80"}, - {"http://reg.example:5000", "reg.example:5000"}, - } - for _, tt := range matches { - t.Run(tt.configured+" "+tt.namespace, func(t *testing.T) { - h := NewContainerHandlerWithRegistry(nil, nsTestProxyURL, "", map[string]string{"lab": tt.configured}) - registryURL, upstreamName, cacheName, ok := h.registryForNamespace(tt.namespace, "owner/app") - if !ok { - t.Fatalf("ns %q did not resolve for upstream %q", tt.namespace, tt.configured) - } - if registryURL != strings.TrimSuffix(tt.configured, "/") || upstreamName != "owner/app" || cacheName != "upstream/lab/owner/app" { - t.Errorf("route = (%q, %q, %q), want (%q, owner/app, upstream/lab/owner/app)", - registryURL, upstreamName, cacheName, tt.configured) - } - }) - } - - mismatches := []struct { - configured string - namespace string - }{ - {"http://reg.example:5000", "reg.example:5001"}, - {"https://reg.example:80", "reg.example"}, - {"https://reg.example", "reg.example:80"}, - {"http://reg.example:443", "reg.example"}, - } - for _, tt := range mismatches { - t.Run("mismatch "+tt.configured+" "+tt.namespace, func(t *testing.T) { - h := NewContainerHandlerWithRegistry(nil, nsTestProxyURL, "", map[string]string{"lab": tt.configured}) - if _, _, _, ok := h.registryForNamespace(tt.namespace, "owner/app"); ok { - t.Errorf("ns %q resolved for upstream %q, want no match", tt.namespace, tt.configured) - } - }) - } -} - -func TestContainerHandler_NamespaceKeepsNonDefaultPorts(t *testing.T) { - // Two registries on one host that differ only in the port, one of them on - // the scheme-default port. Tests cannot listen on ports 80 or 443, so the - // dialer maps those addresses to the fake registries. - onDefaultPort := newNSTestRegistry(t, "owner/app", "") - onOtherPort := newNSTestRegistry(t, "owner/app", "") - routes, h, _ := newNSTestHandler(t, "", map[string]string{ - "std": "http://registry.example", - "alt": "http://registry.example:443", - }) - endpoints := map[string]string{ - "registry.example:80": onDefaultPort.Listener.Addr().String(), - "registry.example:443": onOtherPort.Listener.Addr().String(), - } - transport := &http.Transport{DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - target, ok := endpoints[addr] - if !ok { - return nil, fmt.Errorf("unexpected dial to %s", addr) - } - return (&net.Dialer{}).DialContext(ctx, network, target) - }} - t.Cleanup(transport.CloseIdleConnections) - h.proxy.HTTPClient = &http.Client{Transport: transport} - - // Every request below misses the cache: a different registry or a - // different endpoint than the request before it. - tests := []struct { - target string - wantDefault int - wantOther int - }{ - {"/v2/owner/app/manifests/latest?ns=registry.example:80", 1, 0}, - {"/v2/owner/app/manifests/latest?ns=registry.example:443", 1, 1}, - {"/v2/owner/app/tags/list?ns=registry.example", 2, 1}, - } - for _, tt := range tests { - response := serveNS(routes, tt.target) - if response.Code != http.StatusOK { - t.Fatalf("%s status = %d: %s", tt.target, response.Code, response.Body.String()) - } - if onDefaultPort.requestCount() != tt.wantDefault || onOtherPort.requestCount() != tt.wantOther { - t.Errorf("after %s: requests default-port=%d other-port=%d, want %d/%d", - tt.target, onDefaultPort.requestCount(), onOtherPort.requestCount(), tt.wantDefault, tt.wantOther) - } - } -} - -func TestContainerHandler_NamespaceHostCollisions(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - logs := &bytes.Buffer{} - proxy.Logger = slog.New(slog.NewTextHandler(logs, nil)) - h := NewContainerHandlerWithRegistry(proxy, nsTestProxyURL, "https://mirror.example", map[string]string{ - "zeta": "https://shared.example", - "alpha": "https://Shared.example:443", - "mirror": "https://mirror.example", - "hub": "https://registry-1.docker.io", - }) - - tests := map[string]string{ - "shared.example": "https://Shared.example:443", - "mirror.example": "https://mirror.example", - "registry-1.docker.io": "https://mirror.example", - } - for namespace, want := range tests { - registryURL, _, cacheName, ok := h.registryForNamespace(namespace, "owner/app") - if !ok || registryURL != want { - t.Errorf("ns %q resolved to (%q, %v), want %q", namespace, registryURL, ok, want) - } - if namespace == "shared.example" && cacheName != "upstream/alpha/owner/app" { - t.Errorf("ns %q cache name = %q, want upstream/alpha/owner/app", namespace, cacheName) - } - } - for _, upstream := range []string{"upstream=zeta", "upstream=mirror", "upstream=hub"} { - if !strings.Contains(logs.String(), upstream) { - t.Errorf("missing collision warning for %s in logs:\n%s", upstream, logs.String()) - } - } -} - -func TestContainerHandler_NamespaceCollisionWarningNamesEachOwner(t *testing.T) { - // r's two keys end up with two different owners; the warning has to - // name both, or an admin fixing one entry misses the other. - proxy, _, _, _ := setupTestProxy(t) - logs := &bytes.Buffer{} - proxy.Logger = slog.New(slog.NewTextHandler(logs, nil)) - h := NewContainerHandlerWithRegistry(proxy, nsTestProxyURL, "", map[string]string{ - "p": "http://h.example", - "q": "http://h.example:443", - "r": "https://h.example", - }) - - for namespace, want := range map[string]string{"h.example": "p", "h.example:80": "p", "h.example:443": "q"} { - if got := h.namespaces[namespace]; got != want { - t.Errorf("index[%q] = %q, want %q", namespace, got, want) - } - } - for _, want := range []string{"upstream=r", "h.example=p", "h.example:443=q"} { - if !strings.Contains(logs.String(), want) { - t.Errorf("logs missing %q:\n%s", want, logs.String()) - } - } -} - -func TestContainerHandler_NamespaceSkipsRegistryURLsWithPath(t *testing.T) { - registry := newNSTestRegistry(t, "owner/app", "/artifactory/api/docker/remote") - routes, _, logs := newNSTestHandler(t, "", map[string]string{ - "art": registry.URL + "/artifactory/api/docker/remote", - }) - - assertNameUnknown(t, serveNS(routes, "/v2/owner/app/manifests/latest?ns="+registry.host())) - if got := registry.requestCount(); got != 0 { - t.Errorf("upstream requests via ns = %d, want 0", got) - } - if !strings.Contains(logs.String(), "upstream=art") { - t.Errorf("missing warning for path-prefixed upstream in logs:\n%s", logs.String()) - } - - // Per-registry mirrors with override_path reach it with ns attached. - if response := serveNS(routes, "/v2/upstream/art/owner/app/manifests/latest?ns="+registry.host()); response.Code != http.StatusOK { - t.Fatalf("prefix route with ns status = %d, want 200: %s", response.Code, response.Body.String()) - } - if got := registry.requestCount(); got != 1 { - t.Errorf("upstream requests via prefix route with ns = %d, want 1", got) - } - if response := serveNS(routes, "/v2/upstream/art/owner/app/manifests/latest"); response.Code != http.StatusOK { - t.Fatalf("prefix route status = %d, want 200: %s", response.Code, response.Body.String()) - } - - t.Run("default registry", func(t *testing.T) { - mirror := newNSTestRegistry(t, "library/nginx", "/hub") - routes, _, logs := newNSTestHandler(t, mirror.URL+"/hub", nil) - - assertNameUnknown(t, serveNS(routes, "/v2/library/nginx/manifests/latest?ns="+mirror.host())) - if got := mirror.requestCount(); got != 0 { - t.Errorf("upstream requests via ns host = %d, want 0", got) - } - if response := serveNS(routes, "/v2/library/nginx/manifests/latest?ns=docker.io"); response.Code != http.StatusOK { - t.Fatalf("ns=docker.io status = %d, want 200: %s", response.Code, response.Body.String()) - } - if !strings.Contains(logs.String(), "default OCI registry") { - t.Errorf("missing warning for path-prefixed default registry in logs:\n%s", logs.String()) - } - }) -} - -func TestContainerHandler_TagsListIgnoresLegacyCacheRows(t *testing.T) { - registry := newNSTestRegistry(t, "owner/app", "") - routes, h, _ := newNSTestHandler(t, "", map[string]string{"ghcr": registry.URL}) - - // Rows written before the raw-link format hold a Link already rewritten - // for the route that filled them. Seed one under the legacy identity. - query := url.Values{"n": {"1"}} - legacySum := sha256.Sum256([]byte(registry.URL + "\x00owner/app\x00" + query.Encode())) - legacyKey := hex.EncodeToString(legacySum[:]) - if legacyKey == h.containerTagsCacheKey(registry.URL, "owner/app", query) { - t.Fatal("legacy and current tag-list cache keys are equal") - } - legacy := &cachedContainerTags{ - body: []byte(`{"name":"owner/app","tags":["legacy"]}`), - contentType: contentTypeJSON, - link: `<` + nsTestProxyURL + `/v2/upstream/ghcr/owner/app/tags/list?last=legacy&n=1>; rel="next"`, - fetchedAt: time.Now(), - } - if err := h.storeContainerTags(context.Background(), legacyKey, legacy); err != nil { - t.Fatalf("store legacy tag list: %v", err) - } - - response := serveNS(routes, "/v2/owner/app/tags/list?n=1&ns="+registry.host()) - if response.Code != http.StatusOK { - t.Fatalf("status = %d: %s", response.Code, response.Body.String()) - } - if got, want := response.Body.String(), `{"name":"owner/app","tags":["1.0"]}`; got != want { - t.Errorf("body = %q, want %q (legacy row must not be served)", got, want) - } - if got := registry.requestCount(); got != 1 { - t.Errorf("upstream requests = %d, want 1 (legacy row must not be served)", got) - } -} - -func TestContainerHandler_NamespaceWarningsRedactCredentials(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - logs := &bytes.Buffer{} - proxy.Logger = slog.New(slog.NewTextHandler(logs, nil)) - NewContainerHandlerWithRegistry(proxy, nsTestProxyURL, "https://svc:s3cret@mirror.example/hub", map[string]string{ - "art": "https://bot:hunter2@art.example/artifactory/api/docker/remote", - }) - - for _, secret := range []string{"s3cret", "hunter2"} { - if strings.Contains(logs.String(), secret) { - t.Errorf("logs contain credential %q:\n%s", secret, logs.String()) - } - } - for _, want := range []string{ - "svc:xxxxx@mirror.example/hub", - "bot:xxxxx@art.example/artifactory", - "Docker Hub aliases still select it", - } { - if !strings.Contains(logs.String(), want) { - t.Errorf("logs missing %q:\n%s", want, logs.String()) - } - } -} diff --git a/internal/handler/container_referrers.go b/internal/handler/container_referrers.go deleted file mode 100644 index 94904f8..0000000 --- a/internal/handler/container_referrers.go +++ /dev/null @@ -1,222 +0,0 @@ -package handler - -import ( - "cmp" - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "regexp" - "strconv" - "strings" - "time" -) - -const ( - containerReferrersCacheEcosystem = "oci-referrers" - containerReferrersMediaType = "application/vnd.oci.image.index.v1+json" - referrersMatchCount = 3 // full match + name + digest -) - -var ( - // referrersPathPattern matches referrers paths: {name}/referrers/{digest} - referrersPathPattern = regexp.MustCompile(`^(.+)/referrers/([^/]+)$`) - referrersDigestPattern = regexp.MustCompile(`^(sha256:[a-f0-9]{64}|sha512:[a-f0-9]{128})$`) -) - -// parseReferrersPath extracts repository name and subject digest from a -// referrers path. -func (h *ContainerHandler) parseReferrersPath(path string) (name, digest string) { - matches := referrersPathPattern.FindStringSubmatch(path) - if len(matches) != referrersMatchCount { - return "", "" - } - return matches[1], matches[2] -} - -// handleReferrers serves the OCI 1.1 referrers API. -// Path format: {name}/referrers/{digest} -func (h *ContainerHandler) handleReferrers(w http.ResponseWriter, r *http.Request, path string) { - if r.Method != http.MethodGet { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - - name, digest := h.parseReferrersPath(path) - if name == "" || !referrersDigestPattern.MatchString(digest) { - h.containerError(w, http.StatusBadRequest, "DIGEST_INVALID", "invalid referrers digest") - return - } - - registryURL, upstreamName, _, ok := h.registryForRequest(r, name) - if !ok { - h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry") - return - } - - h.proxy.Logger.Debug("container referrers request", "name", upstreamName, "digest", digest) - h.serveReferrers(w, r, registryURL, upstreamName, digest) -} - -func (h *ContainerHandler) serveReferrers(w http.ResponseWriter, r *http.Request, registryURL, name, digest string) { - // artifactType is never forwarded, so the upstream returns the full index - // and one row serves every filter. Without OCI-Filters-Applied in the - // response, clients filter the index themselves, as the spec requires. - // ns only picks the registry, so it is neither forwarded nor part of the - // cache identity. - query := r.URL.Query() - query.Del("artifactType") - query.Del(namespaceQueryParam) - // Same identity shape as manifests (registry, name, reference, variant); - // the oci-referrers ecosystem keeps the rows apart. - cacheKey := h.containerManifestCacheKey(registryURL, name, digest, query.Encode()) - cached, err := h.loadContainerReferrers(r.Context(), cacheKey) - if err != nil { - h.proxy.Logger.Warn("failed to read cached container referrers", "error", err) - cached = nil - } - if cached != nil && h.containerTagsFresh(cached) { - h.writeContainerReferrers(w, r, registryURL, cached, false) - return - } - - upstreamURL := fmt.Sprintf("%s/v2/%s/referrers/%s", registryURL, name, digest) - if encoded := query.Encode(); encoded != "" { - upstreamURL += "?" + encoded - } - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) - if err != nil { - h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request") - return - } - req.Header.Set("Accept", containerReferrersMediaType) - if cached != nil && cached.etag != "" { - req.Header.Set("If-None-Match", cached.etag) - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - h.serveStaleReferrersOrError(w, r, registryURL, cached, err) - return - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode == http.StatusNotModified && cached != nil { - cached.fetchedAt = time.Now() - if err := h.storeContainerReferrers(r.Context(), cacheKey, cached); err != nil { - h.proxy.Logger.Warn("failed to refresh cached container referrers", "error", err) - } - h.writeContainerReferrers(w, r, registryURL, cached, false) - return - } - if resp.StatusCode != http.StatusOK { - if shouldServeStaleManifest(resp.StatusCode) && cached != nil { - h.serveStaleReferrersOrError(w, r, registryURL, cached, - fmt.Errorf("upstream returned status %d", resp.StatusCode)) - return - } - // Upstream errors reach the client as sent. Only a 404 tells it the - // registry has no referrers API, so it falls back to the sha256- - // tag schema served as manifests. - h.proxy.relayResponse(w, r, resp, copyContainerReferrersHeaders) - return - } - - body, err := h.proxy.ReadMetadata(resp.Body) - if err != nil { - h.serveStaleReferrersOrError(w, r, registryURL, cached, fmt.Errorf("reading referrers: %w", err)) - return - } - if !json.Valid(body) { - h.serveStaleReferrersOrError(w, r, registryURL, cached, errors.New("upstream referrers response is not JSON")) - return - } - referrers := &cachedContainerTags{ - body: body, - contentType: resp.Header.Get(headerContentType), - etag: resp.Header.Get(headerETag), - // Stored as sent upstream and rewritten when served, so a row shared - // by several client paths to the same registry links back correctly. - link: strings.Join(resp.Header.Values("Link"), ", "), - size: int64(len(body)), - fetchedAt: time.Now(), - } - if referrers.contentType == "" { - referrers.contentType = containerReferrersMediaType - } - if err := h.storeContainerReferrers(r.Context(), cacheKey, referrers); err != nil { - h.proxy.Logger.Warn("failed to cache container referrers", "error", err) - } - h.writeContainerReferrers(w, r, registryURL, referrers, false) -} - -// serveStaleReferrersOrError serves a cached index when the upstream cannot -// answer. Without one it returns 502 rather than 404, so clients do not take -// a failed fetch as a registry without the referrers API. -func (h *ContainerHandler) serveStaleReferrersOrError(w http.ResponseWriter, r *http.Request, registryURL string, cached *cachedContainerTags, err error) { - if cached != nil { - h.proxy.Logger.Warn("upstream referrers fetch failed, serving stale cache", "error", err) - h.writeContainerReferrers(w, r, registryURL, cached, true) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("failed to fetch container referrers", "error", err) - h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") -} - -// copyContainerReferrersHeaders also keeps Retry-After, since uncached 429 -// and 5xx answers are relayed here instead of turning into a 502. -func copyContainerReferrersHeaders(destination, source http.Header) { - copyContainerTagsHeaders(destination, source) - if value := source.Get("Retry-After"); value != "" { - destination.Set("Retry-After", value) - } -} - -func (h *ContainerHandler) loadContainerReferrers(ctx context.Context, cacheKey string) (*cachedContainerTags, error) { - entry, body, err := h.loadContainerMetadata(ctx, containerReferrersCacheEcosystem, cacheKey) - if err != nil || entry == nil { - return nil, err - } - referrers := &cachedContainerTags{ - body: body, - contentType: cmp.Or(entry.ContentType.String, containerReferrersMediaType), - etag: entry.ETag.String, - link: entry.Link.String, - size: int64(len(body)), - fetchedAt: entry.FetchedAt.Time, - } - if entry.Size.Valid { - referrers.size = entry.Size.Int64 - } - return referrers, nil -} - -func (h *ContainerHandler) storeContainerReferrers(ctx context.Context, cacheKey string, referrers *cachedContainerTags) error { - size, err := h.storeContainerMetadata(ctx, containerReferrersCacheEcosystem, cacheKey, referrers.body, - referrers.etag, referrers.link, referrers.contentType, "", time.Time{}, referrers.fetchedAt) - if err != nil { - return fmt.Errorf("storing referrers: %w", err) - } - referrers.size = size - return nil -} - -func (h *ContainerHandler) writeContainerReferrers(w http.ResponseWriter, r *http.Request, registryURL string, referrers *cachedContainerTags, stale bool) { - w.Header().Set(headerContentType, referrers.contentType) - w.Header().Set(headerContentLength, strconv.FormatInt(referrers.size, 10)) - if referrers.etag != "" { - w.Header().Set(headerETag, referrers.etag) - } - if link := h.rewriteContainerTagsLink(referrers.link, registryURL, r.URL.Path, r.URL.Query().Get(namespaceQueryParam)); link != "" { - w.Header().Set("Link", link) - } - if stale { - w.Header().Set("Warning", containerStaleWarning) - } - w.WriteHeader(http.StatusOK) - _, _ = w.Write(referrers.body) -} diff --git a/internal/handler/container_referrers_test.go b/internal/handler/container_referrers_test.go deleted file mode 100644 index 77ef5c8..0000000 --- a/internal/handler/container_referrers_test.go +++ /dev/null @@ -1,552 +0,0 @@ -package handler - -import ( - "io" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "testing" - "time" -) - -const testReferrersIndex = `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` + - `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:1111111111111111111111111111111111111111111111111111111111111111","size":10,"artifactType":"application/spdx+json"},` + - `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:2222222222222222222222222222222222222222222222222222222222222222","size":10,"artifactType":"application/vnd.cncf.notary.signature"}]}` - -var testReferrersSubject = "sha256:" + sha256Hex("subject manifest") - -func newReferrersTestHandler(t *testing.T, upstream *httptest.Server) (*ContainerHandler, *Proxy) { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - return &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://proxy.example.test"}, proxy -} - -func serveReferrersRequest(h *ContainerHandler, method, target string) *httptest.ResponseRecorder { - recorder := httptest.NewRecorder() - h.Routes().ServeHTTP(recorder, httptest.NewRequest(method, target, nil)) - return recorder -} - -func TestContainerHandler_ReferrersCachesIndexForEveryFilter(t *testing.T) { - upstreamAvailable := true - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - if r.URL.Path != "/v2/library/nginx/referrers/"+testReferrersSubject { - http.NotFound(w, r) - return - } - if r.URL.RawQuery != "" { - t.Errorf("upstream query = %q, want artifactType stripped", r.URL.RawQuery) - } - if got := r.Header.Get("Accept"); got != containerReferrersMediaType { - t.Errorf("upstream Accept = %q, want %q", got, containerReferrersMediaType) - } - if !upstreamAvailable { - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", containerReferrersMediaType) - w.Header().Set("OCI-Filters-Applied", "artifactType") - _, _ = io.WriteString(w, testReferrersIndex) - })) - defer upstream.Close() - - h, proxy := newReferrersTestHandler(t, upstream) - proxy.MetadataTTL = time.Hour - base := "/library/nginx/referrers/" + testReferrersSubject - - first := serveReferrersRequest(h, http.MethodGet, base+"?artifactType=application/spdx%2Bjson") - if first.Code != http.StatusOK { - t.Fatalf("first status = %d, want 200: %s", first.Code, first.Body.String()) - } - if first.Body.String() != testReferrersIndex { - t.Errorf("first body = %q, want upstream index", first.Body.String()) - } - if got := first.Header().Get("Content-Type"); got != containerReferrersMediaType { - t.Errorf("first Content-Type = %q, want %q", got, containerReferrersMediaType) - } - if got := first.Header().Get("OCI-Filters-Applied"); got != "" { - t.Errorf("OCI-Filters-Applied = %q, want absent so clients filter", got) - } - - other := serveReferrersRequest(h, http.MethodGet, base+"?artifactType=application/vnd.cncf.notary.signature") - if other.Code != http.StatusOK || other.Body.String() != testReferrersIndex { - t.Fatalf("other filter = %d %q, want cached full index", other.Code, other.Body.String()) - } - if upstreamRequests != 1 { - t.Fatalf("upstream requests after fresh hit = %d, want 1", upstreamRequests) - } - - proxy.MetadataTTL = 0 - upstreamAvailable = false - stale := serveReferrersRequest(h, http.MethodGet, base) - if stale.Code != http.StatusOK || stale.Body.String() != testReferrersIndex { - t.Fatalf("stale = %d %q, want cached index", stale.Code, stale.Body.String()) - } - if got := stale.Header().Get("Content-Type"); got != containerReferrersMediaType { - t.Errorf("stale Content-Type = %q, want %q", got, containerReferrersMediaType) - } - if got := stale.Header().Get("Warning"); got != containerStaleWarning { - t.Errorf("stale Warning = %q, want stale warning", got) - } - if upstreamRequests != 2 { - t.Errorf("upstream requests after stale fallback = %d, want 2", upstreamRequests) - } -} - -func TestContainerHandler_ReferrersRevalidatesWithETag(t *testing.T) { - upstreamRequests := 0 - notModified := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - if r.Header.Get("If-None-Match") == `"referrers-etag"` { - notModified++ - w.WriteHeader(http.StatusNotModified) - return - } - w.Header().Set("Content-Type", containerReferrersMediaType) - w.Header().Set("ETag", `"referrers-etag"`) - _, _ = io.WriteString(w, testReferrersIndex) - })) - defer upstream.Close() - - h, proxy := newReferrersTestHandler(t, upstream) - proxy.MetadataTTL = time.Hour - target := "/library/nginx/referrers/" + testReferrersSubject - - if first := serveReferrersRequest(h, http.MethodGet, target); first.Code != http.StatusOK { - t.Fatalf("first status = %d, want 200", first.Code) - } - // Age the row past the TTL so the next request has to revalidate. - cacheKey := h.containerManifestCacheKey(upstream.URL, "library/nginx", testReferrersSubject, "") - entry, err := proxy.DB.GetMetadataCache(containerReferrersCacheEcosystem, cacheKey) - if err != nil || entry == nil { - t.Fatalf("cached row = %v, %v, want one", entry, err) - } - entry.FetchedAt.Time = time.Now().Add(-2 * time.Hour) - if err := proxy.DB.UpsertMetadataCache(entry); err != nil { - t.Fatalf("ageing cached row: %v", err) - } - - second := serveReferrersRequest(h, http.MethodGet, target) - if second.Code != http.StatusOK || second.Body.String() != testReferrersIndex { - t.Fatalf("revalidated = %d %q, want cached index", second.Code, second.Body.String()) - } - if got := second.Header().Get("Warning"); got != "" { - t.Errorf("Warning = %q, want none after 304", got) - } - - // The 304 starts the TTL again, so this one stays in the cache. - if third := serveReferrersRequest(h, http.MethodGet, target); third.Code != http.StatusOK { - t.Fatalf("third status = %d, want 200", third.Code) - } - if upstreamRequests != 2 || notModified != 1 { - t.Errorf("upstream requests = %d with %d answered 304, want 2 with 1", upstreamRequests, notModified) - } -} - -func TestContainerHandler_ReferrersDefaultsMissingContentType(t *testing.T) { - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - upstreamRequests++ - // A nil value stops net/http from sniffing a Content-Type. - w.Header()["Content-Type"] = nil - _, _ = io.WriteString(w, testReferrersIndex) - })) - defer upstream.Close() - - h, proxy := newReferrersTestHandler(t, upstream) - proxy.MetadataTTL = time.Hour - target := "/library/nginx/referrers/" + testReferrersSubject - - // oras-go only accepts the exact image index type, both fresh and cached. - for _, label := range []string{"fetched", "cached"} { - got := serveReferrersRequest(h, http.MethodGet, target) - if got.Code != http.StatusOK { - t.Fatalf("%s status = %d, want 200", label, got.Code) - } - if ct := got.Header().Get("Content-Type"); ct != containerReferrersMediaType { - t.Errorf("%s Content-Type = %q, want %q", label, ct, containerReferrersMediaType) - } - } - if upstreamRequests != 1 { - t.Errorf("upstream requests = %d, want 1", upstreamRequests) - } -} - -func TestContainerHandler_ReferrersRelaysUpstreamErrorsWithoutCaching(t *testing.T) { - tests := []struct { - name string - status int - header string - value string - wantError string - }{ - {name: "no referrers API", status: http.StatusNotFound, header: "Content-Type", value: "application/json", wantError: "NOT_FOUND"}, - {name: "repository unknown", status: http.StatusNotFound, header: "Content-Type", value: "application/json", wantError: "NAME_UNKNOWN"}, - {name: "auth challenge", status: http.StatusUnauthorized, header: "WWW-Authenticate", value: `Bearer realm="https://auth.example.test/token"`, wantError: "UNAUTHORIZED"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - upstreamRequests++ - w.Header().Set(tt.header, tt.value) - w.WriteHeader(tt.status) - _, _ = io.WriteString(w, `{"errors":[{"code":"`+tt.wantError+`"}]}`) - })) - defer upstream.Close() - - h, proxy := newReferrersTestHandler(t, upstream) - proxy.MetadataTTL = time.Hour - target := "/library/nginx/referrers/" + testReferrersSubject - for range 2 { - got := serveReferrersRequest(h, http.MethodGet, target) - if got.Code != tt.status { - t.Fatalf("status = %d, want %d", got.Code, tt.status) - } - if !strings.Contains(got.Body.String(), tt.wantError) { - t.Errorf("body = %q, want upstream error %s", got.Body.String(), tt.wantError) - } - if got.Header().Get(tt.header) != tt.value { - t.Errorf("%s = %q, want %q", tt.header, got.Header().Get(tt.header), tt.value) - } - } - if upstreamRequests != 2 { - t.Errorf("upstream requests = %d, want 2 (error must not be cached)", upstreamRequests) - } - }) - } -} - -func TestContainerHandler_ReferrersReportsUpstreamFailureWithoutCache(t *testing.T) { - tests := []struct { - name string - handler http.HandlerFunc - wantStatus int - wantRetryAfter string - }{ - {name: "upstream 503", wantStatus: http.StatusServiceUnavailable, wantRetryAfter: "120", handler: func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Retry-After", "120") - http.Error(w, "unavailable", http.StatusServiceUnavailable) - }}, - {name: "upstream 429", wantStatus: http.StatusTooManyRequests, wantRetryAfter: "60", handler: func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Retry-After", "60") - http.Error(w, "slow down", http.StatusTooManyRequests) - }}, - {name: "upstream not JSON", wantStatus: http.StatusBadGateway, handler: func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "text/html") - _, _ = io.WriteString(w, "captive portal") - }}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - tt.handler(w, r) - })) - defer upstream.Close() - - h, proxy := newReferrersTestHandler(t, upstream) - proxy.MetadataTTL = time.Hour - target := "/library/nginx/referrers/" + testReferrersSubject - for range 2 { - got := serveReferrersRequest(h, http.MethodGet, target) - if got.Code != tt.wantStatus { - t.Fatalf("status = %d, want %d: %s", got.Code, tt.wantStatus, got.Body.String()) - } - if got.Header().Get("Retry-After") != tt.wantRetryAfter { - t.Errorf("Retry-After = %q, want %q", got.Header().Get("Retry-After"), tt.wantRetryAfter) - } - } - if upstreamRequests != 2 { - t.Errorf("upstream requests = %d, want 2 (failure must not be cached)", upstreamRequests) - } - }) - } - - t.Run("upstream unreachable", func(t *testing.T) { - upstream := httptest.NewServer(http.NotFoundHandler()) - h, _ := newReferrersTestHandler(t, upstream) - upstream.Close() - got := serveReferrersRequest(h, http.MethodGet, "/library/nginx/referrers/"+testReferrersSubject) - if got.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want 502 so clients keep using the referrers API: %s", got.Code, got.Body.String()) - } - }) -} - -func TestContainerHandler_ReferrersNamedRegistryRewritesLink(t *testing.T) { - upstreamRequests := 0 - var upstream *httptest.Server - upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - if r.URL.Path != "/v2/owner/img/referrers/"+testReferrersSubject { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", containerReferrersMediaType) - switch r.URL.Query().Get("last") { - case "": - w.Header().Add("Link", `; rel="next"`) - w.Header().Add("Link", `; rel="help"`) - case "abc": - w.Header().Set("Link", `<`+upstream.URL+`/v2/owner/img/referrers/`+testReferrersSubject+`?last=def>; rel="next"`) - } - _, _ = io.WriteString(w, testReferrersIndex) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.MetadataTTL = time.Hour - // The default registry is the same upstream, so upstream/test/owner/img - // and owner/img share one cache row. - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", upstream.URL, map[string]string{"test": upstream.URL}) - routes := http.StripPrefix("/v2", h.Routes()) - - first := httptest.NewRecorder() - routes.ServeHTTP(first, httptest.NewRequest(http.MethodGet, - "/v2/upstream/test/owner/img/referrers/"+testReferrersSubject+"?artifactType=application/spdx%2Bjson", nil)) - if first.Code != http.StatusOK { - t.Fatalf("first page status = %d, want 200: %s", first.Code, first.Body.String()) - } - wantLink := `; rel="next", ; rel="help"` - if got := first.Header().Get("Link"); got != wantLink { - t.Fatalf("Link = %q, want %q", got, wantLink) - } - - shared := httptest.NewRecorder() - routes.ServeHTTP(shared, httptest.NewRequest(http.MethodGet, "/v2/owner/img/referrers/"+testReferrersSubject, nil)) - if shared.Code != http.StatusOK { - t.Fatalf("shared row status = %d, want 200: %s", shared.Code, shared.Body.String()) - } - wantSharedLink := `; rel="next", ; rel="help"` - if got := shared.Header().Get("Link"); got != wantSharedLink { - t.Errorf("shared row Link = %q, want it rewritten for the second path %q", got, wantSharedLink) - } - if upstreamRequests != 1 { - t.Fatalf("upstream requests after shared row hit = %d, want 1", upstreamRequests) - } - - nextURL := strings.TrimPrefix(strings.SplitN(first.Header().Get("Link"), ">", 2)[0], "<") - next := httptest.NewRecorder() - routes.ServeHTTP(next, httptest.NewRequest(http.MethodGet, nextURL, nil)) - if next.Code != http.StatusOK { - t.Fatalf("next page status = %d, want 200: %s", next.Code, next.Body.String()) - } - wantNextLink := `; rel="next"` - if got := next.Header().Get("Link"); got != wantNextLink { - t.Errorf("next page Link = %q, want absolute upstream link rewritten to %q", got, wantNextLink) - } - if upstreamRequests != 2 { - t.Errorf("upstream requests = %d, want 2 (pages are separate rows)", upstreamRequests) - } -} - -func TestContainerHandler_ReferrersRejectsInvalidRequests(t *testing.T) { - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - upstreamRequests++ - w.WriteHeader(http.StatusInternalServerError) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", upstream.URL) - - tests := []struct { - name string - method string - path string - wantStatus int - wantCode string - }{ - {name: "post", method: http.MethodPost, path: "/library/nginx/referrers/" + testReferrersSubject, wantStatus: http.StatusMethodNotAllowed}, - {name: "short digest", method: http.MethodGet, path: "/library/nginx/referrers/sha256:abc", wantStatus: http.StatusBadRequest, wantCode: "DIGEST_INVALID"}, - {name: "trailing characters", method: http.MethodGet, path: "/library/nginx/referrers/" + testReferrersSubject + "x", wantStatus: http.StatusBadRequest, wantCode: "DIGEST_INVALID"}, - {name: "leading characters", method: http.MethodGet, path: "/library/nginx/referrers/x" + testReferrersSubject, wantStatus: http.StatusBadRequest, wantCode: "DIGEST_INVALID"}, - {name: "unknown algorithm", method: http.MethodGet, path: "/library/nginx/referrers/md5:" + strings.Repeat("a", 32), wantStatus: http.StatusBadRequest, wantCode: "DIGEST_INVALID"}, - {name: "unknown named upstream", method: http.MethodGet, path: "/upstream/missing/owner/img/referrers/" + testReferrersSubject, wantStatus: http.StatusNotFound, wantCode: "NAME_UNKNOWN"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := serveReferrersRequest(h, tt.method, tt.path) - if got.Code != tt.wantStatus { - t.Fatalf("status = %d, want %d: %s", got.Code, tt.wantStatus, got.Body.String()) - } - if tt.wantCode != "" && !strings.Contains(got.Body.String(), tt.wantCode) { - t.Errorf("body = %q, want %s", got.Body.String(), tt.wantCode) - } - }) - } - if upstreamRequests != 0 { - t.Errorf("upstream requests = %d, want 0", upstreamRequests) - } -} - -func TestContainerHandler_ReferrersRouteKeepsManifestPaths(t *testing.T) { - tagSchema := "sha256-" + strings.TrimPrefix(testReferrersSubject, "sha256:") - var upstreamPaths []string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamPaths = append(upstreamPaths, r.URL.Path) - // Both handlers build the same upstream path here, so tell them apart - // by Accept: only the referrers handler asks for the index type alone. - if r.Header.Get("Accept") == containerReferrersMediaType { - http.Error(w, "reached the referrers handler", http.StatusBadRequest) - return - } - w.Header().Set("Content-Type", containerReferrersMediaType) - _, _ = io.WriteString(w, testReferrersIndex) - })) - defer upstream.Close() - - h, _ := newReferrersTestHandler(t, upstream) - for _, path := range []string{ - // A repository may contain a "referrers" component; manifest paths - // must keep reaching the manifest handler. - "/foo/referrers/app/manifests/latest", - // Clients without a referrers API read the tag schema as a manifest. - "/library/nginx/manifests/" + tagSchema, - } { - if got := serveReferrersRequest(h, http.MethodGet, path); got.Code != http.StatusOK { - t.Fatalf("GET %s status = %d, want 200: %s", path, got.Code, got.Body.String()) - } - } - want := []string{ - "/v2/foo/referrers/app/manifests/latest", - "/v2/library/nginx/manifests/" + tagSchema, - } - if strings.Join(upstreamPaths, "\n") != strings.Join(want, "\n") { - t.Errorf("upstream paths = %q, want %q", upstreamPaths, want) - } -} - -func TestContainerHandler_ReferrersRouteAllowsEndpointNamesInRepository(t *testing.T) { - var upstreamPaths []string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamPaths = append(upstreamPaths, r.URL.Path) - // The manifest handler would forward the same path, so only answer - // the referrers handler, which asks for the index type alone. - if r.Header.Get("Accept") != containerReferrersMediaType { - http.Error(w, "reached the manifest handler", http.StatusBadRequest) - return - } - w.Header().Set("Content-Type", containerReferrersMediaType) - _, _ = io.WriteString(w, testReferrersIndex) - })) - defer upstream.Close() - - h, _ := newReferrersTestHandler(t, upstream) - for _, name := range []string{"owner/tags/list", "owner/manifests/app"} { - upstreamPaths = nil - got := serveReferrersRequest(h, http.MethodGet, "/"+name+"/referrers/"+testReferrersSubject) - if got.Code != http.StatusOK { - t.Fatalf("%s: status = %d, want 200: %s", name, got.Code, got.Body.String()) - } - want := "/v2/" + name + "/referrers/" + testReferrersSubject - if len(upstreamPaths) != 1 || upstreamPaths[0] != want { - t.Errorf("%s: upstream paths = %q, want [%q]", name, upstreamPaths, want) - } - } -} - -func TestContainerHandler_ReferrersFollowsNamespace(t *testing.T) { - var upstreamQueries []string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v2/owner/img/referrers/"+testReferrersSubject { - http.NotFound(w, r) - return - } - upstreamQueries = append(upstreamQueries, r.URL.RawQuery) - w.Header().Set("Content-Type", containerReferrersMediaType) - w.Header().Set("Link", `; rel="next"`) - _, _ = io.WriteString(w, testReferrersIndex) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.MetadataTTL = time.Hour - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", "", map[string]string{"test": upstream.URL}) - routes := http.StripPrefix("/v2", h.Routes()) - host := strings.TrimPrefix(upstream.URL, "http://") - target := "/v2/owner/img/referrers/" + testReferrersSubject - - withNS := httptest.NewRecorder() - routes.ServeHTTP(withNS, httptest.NewRequest(http.MethodGet, target+"?ns="+url.QueryEscape(host), nil)) - if withNS.Code != http.StatusOK { - t.Fatalf("ns request status = %d, want 200: %s", withNS.Code, withNS.Body.String()) - } - wantLink := `; rel="next"` - if got := withNS.Header().Get("Link"); got != wantLink { - t.Errorf("Link = %q, want next page kept on the ns route %q", got, wantLink) - } - - // The prefix route reaches the same registry and shares the row. - prefixed := httptest.NewRecorder() - routes.ServeHTTP(prefixed, httptest.NewRequest(http.MethodGet, "/v2/upstream/test/owner/img/referrers/"+testReferrersSubject, nil)) - if prefixed.Code != http.StatusOK { - t.Fatalf("prefix request status = %d, want 200: %s", prefixed.Code, prefixed.Body.String()) - } - if len(upstreamQueries) != 1 || upstreamQueries[0] != "" { - t.Errorf("upstream queries = %q, want one request without ns", upstreamQueries) - } -} - -// Docker looks up a sha256- referrers tag on every pull. With the -// upstream down and nothing cached, answering it as unknown lets a pull of a -// cached image go ahead; other tags still report the failure. -func TestContainerHandler_ReferrersTagIsUnknownWhenUpstreamFails(t *testing.T) { - referrersTag := "sha256-" + strings.Repeat("ab", 32) - tests := []struct { - name string - reference string - upstream func(t *testing.T) string - want int - }{ - {"referrers tag, upstream 503", referrersTag, upstreamAnswering(http.StatusServiceUnavailable), http.StatusNotFound}, - {"referrers tag, upstream unreachable", referrersTag, unreachableUpstream, http.StatusNotFound}, - {"referrers tag, upstream 404", referrersTag, upstreamAnswering(http.StatusNotFound), http.StatusNotFound}, - {"other tag, upstream 503", "latest", upstreamAnswering(http.StatusServiceUnavailable), http.StatusServiceUnavailable}, - {"other tag, upstream unreachable", "latest", unreachableUpstream, http.StatusBadGateway}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = &http.Client{} - h := &ContainerHandler{proxy: proxy, registryURL: tt.upstream(t), proxyURL: "http://localhost:8080"} - - for _, method := range []string{http.MethodGet, http.MethodHead} { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(method, "/library/nginx/manifests/"+tt.reference, nil)) - if w.Code != tt.want { - t.Errorf("%s status = %d, want %d; body: %s", method, w.Code, tt.want, w.Body.String()) - } - } - }) - } -} - -func upstreamAnswering(status int) func(t *testing.T) string { - return func(t *testing.T) string { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - http.Error(w, http.StatusText(status), status) - })) - t.Cleanup(srv.Close) - return srv.URL - } -} - -func unreachableUpstream(t *testing.T) string { - srv := httptest.NewServer(http.NotFoundHandler()) - srv.Close() - return srv.URL -} diff --git a/internal/handler/container_tags.go b/internal/handler/container_tags.go deleted file mode 100644 index cca97b6..0000000 --- a/internal/handler/container_tags.go +++ /dev/null @@ -1,250 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "net/http" - "net/url" - "regexp" - "strconv" - "strings" - "time" -) - -const ( - containerTagsCacheEcosystem = "oci-tags" - // containerTagsCacheFormat versions the tag-list cache identity. Rows - // written before it hold a Link already rewritten for the route that - // filled them, while this format stores the upstream Link verbatim. The - // two must not share rows: an older binary would serve a raw Link as is. - containerTagsCacheFormat = "raw-link" -) - -var containerLinkTargetPattern = regexp.MustCompile(`<([^>]*)>`) - -type cachedContainerTags struct { - body []byte - contentType string - etag string - link string - size int64 - fetchedAt time.Time -} - -func (h *ContainerHandler) serveTagsList(w http.ResponseWriter, r *http.Request, registryURL, name string) { - // ns only selects the registry; it is neither forwarded upstream nor part - // of the cache identity, so all routes to one registry share tag lists. - query := r.URL.Query() - query.Del(namespaceQueryParam) - cacheKey := h.containerTagsCacheKey(registryURL, name, query) - cached, err := h.loadContainerTags(r.Context(), cacheKey) - if err != nil { - h.proxy.Logger.Warn("failed to read cached container tag list", "error", err) - cached = nil - } - if cached != nil && h.containerTagsFresh(cached) { - h.writeContainerTags(w, r, registryURL, cached, false) - return - } - - upstreamURL := fmt.Sprintf("%s/v2/%s/tags/list", registryURL, name) - if encoded := query.Encode(); encoded != "" { - upstreamURL += "?" + encoded - } - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) - if err != nil { - h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request") - return - } - req.Header.Set("Accept", "application/json") - if cached != nil && cached.etag != "" { - req.Header.Set("If-None-Match", cached.etag) - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - h.serveStaleTagsOrError(w, r, registryURL, cached, err) - return - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode == http.StatusNotModified && cached != nil { - cached.fetchedAt = time.Now() - if err := h.storeContainerTags(r.Context(), cacheKey, cached); err != nil { - h.proxy.Logger.Warn("failed to refresh cached container tag list", "error", err) - } - h.writeContainerTags(w, r, registryURL, cached, false) - return - } - if resp.StatusCode != http.StatusOK { - if cached != nil && shouldServeStaleManifest(resp.StatusCode) { - h.writeContainerTags(w, r, registryURL, cached, true) - return - } - h.proxy.relayResponse(w, r, resp, copyContainerTagsHeaders) - return - } - - body, err := h.proxy.ReadMetadata(resp.Body) - if err != nil { - h.serveStaleTagsOrError(w, r, registryURL, cached, fmt.Errorf("reading tag list: %w", err)) - return - } - // The upstream Link is stored verbatim and rewritten per request because - // clients on different routes share this cache entry. - tags := &cachedContainerTags{ - body: body, - contentType: resp.Header.Get(headerContentType), - etag: resp.Header.Get(headerETag), - link: strings.Join(resp.Header.Values("Link"), ", "), - size: int64(len(body)), - fetchedAt: time.Now(), - } - if tags.contentType == "" { - tags.contentType = contentTypeJSON - } - if err := h.storeContainerTags(r.Context(), cacheKey, tags); err != nil { - h.proxy.Logger.Warn("failed to cache container tag list", "error", err) - } - h.writeContainerTags(w, r, registryURL, tags, false) -} - -func (h *ContainerHandler) serveStaleTagsOrError(w http.ResponseWriter, r *http.Request, registryURL string, cached *cachedContainerTags, err error) { - if cached != nil { - h.proxy.Logger.Warn("upstream tag list fetch failed, serving stale cache", "error", err) - h.writeContainerTags(w, r, registryURL, cached, true) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("failed to fetch container tag list", "error", err) - h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") -} - -func (h *ContainerHandler) containerTagsCacheKey(registryURL, name string, query url.Values) string { - identity := containerTagsCacheFormat + "\x00" + registryURL + "\x00" + name + "\x00" + query.Encode() - sum := sha256.Sum256([]byte(identity)) - return hex.EncodeToString(sum[:]) -} - -func (h *ContainerHandler) containerTagsFresh(tags *cachedContainerTags) bool { - return h.proxy.MetadataTTL > 0 && !tags.fetchedAt.IsZero() && time.Since(tags.fetchedAt) < h.proxy.MetadataTTL -} - -func (h *ContainerHandler) loadContainerTags(ctx context.Context, cacheKey string) (*cachedContainerTags, error) { - if h.proxy.DB == nil || h.proxy.Storage == nil { - return nil, nil - } - entry, err := h.proxy.DB.GetMetadataCache(containerTagsCacheEcosystem, cacheKey) - if err != nil || entry == nil { - return nil, err - } - reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) - if err != nil { - return nil, nil - } - defer func() { _ = reader.Close() }() - body, err := h.proxy.ReadMetadata(reader) - if err != nil { - return nil, err - } - h.proxy.recordMetadataRead(entry) - - tags := &cachedContainerTags{body: body, contentType: contentTypeJSON, size: int64(len(body))} - if entry.ContentType.Valid { - tags.contentType = entry.ContentType.String - } - if entry.ETag.Valid { - tags.etag = entry.ETag.String - } - if entry.Link.Valid { - tags.link = entry.Link.String - } - if entry.Size.Valid { - tags.size = entry.Size.Int64 - } - if entry.FetchedAt.Valid { - tags.fetchedAt = entry.FetchedAt.Time - } - return tags, nil -} - -func (h *ContainerHandler) storeContainerTags(ctx context.Context, cacheKey string, tags *cachedContainerTags) error { - size, err := h.storeContainerMetadata(ctx, containerTagsCacheEcosystem, cacheKey, tags.body, - tags.etag, tags.link, tags.contentType, "", time.Time{}, tags.fetchedAt) - if err != nil { - return fmt.Errorf("storing tag list: %w", err) - } - tags.size = size - return nil -} - -func (h *ContainerHandler) writeContainerTags(w http.ResponseWriter, r *http.Request, registryURL string, tags *cachedContainerTags, stale bool) { - w.Header().Set(headerContentType, tags.contentType) - w.Header().Set(headerContentLength, strconv.FormatInt(tags.size, 10)) - if tags.etag != "" { - w.Header().Set(headerETag, tags.etag) - } - link := h.rewriteContainerTagsLink(tags.link, registryURL, r.URL.Path, r.URL.Query().Get(namespaceQueryParam)) - if link != "" { - w.Header().Set("Link", link) - } - if stale { - w.Header().Set("Warning", containerStaleWarning) - } - w.WriteHeader(http.StatusOK) - _, _ = w.Write(tags.body) -} - -func copyContainerTagsHeaders(destination, source http.Header) { - for _, header := range []string{headerContentType, headerContentLength, headerETag, "Link", "WWW-Authenticate"} { - if value := source.Get(header); value != "" { - destination.Set(header, value) - } - } -} - -func (h *ContainerHandler) rewriteContainerTagsLink(link, registryURL, requestPath, namespace string) string { - if link == "" { - return "" - } - upstreamURL, err := url.Parse(registryURL) - if err != nil { - return link - } - proxyURL, err := url.Parse(h.proxyURL) - if err != nil { - return link - } - - return containerLinkTargetPattern.ReplaceAllStringFunc(link, func(target string) string { - linkURL, err := url.Parse(target[1 : len(target)-1]) - if err != nil { - return target - } - if linkURL.IsAbs() { - if linkURL.Scheme != upstreamURL.Scheme || linkURL.Host != upstreamURL.Host { - return target - } - } else if linkURL.Host != "" || (linkURL.Path != "" && !strings.HasPrefix(linkURL.Path, "/v2/")) { - return target - } - // Relative registry API links resolve against the current tag-list - // endpoint. Rebuild them below so named-registry selectors are kept. - linkURL.Scheme = proxyURL.Scheme - linkURL.Host = proxyURL.Host - linkURL.User = proxyURL.User - linkURL.Path = strings.TrimSuffix(proxyURL.Path, "/") + "/v2" + requestPath - linkURL.RawPath = "" - if namespace != "" { - // Keep follow-up pages on the ns route the client is using. - query := linkURL.Query() - query.Set(namespaceQueryParam, namespace) - linkURL.RawQuery = query.Encode() - } - return "<" + linkURL.String() + ">" - }) -} diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index fd20e38..853059e 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -1,28 +1,19 @@ package handler import ( - "crypto/sha512" - "encoding/hex" + "bytes" + "context" "encoding/json" "io" + "log/slog" "net/http" "net/http/httptest" - "strconv" - "strings" "testing" - "time" "github.com/git-pkgs/proxy/internal/database" - upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" - "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) -var ( - testSHA256Digest = "sha256:" + strings.Repeat("ab", 32) - testSHA512Digest = "sha512:" + strings.Repeat("cd", 64) -) - func TestContainerHandler_parseBlobPath(t *testing.T) { h := &ContainerHandler{} @@ -31,30 +22,20 @@ func TestContainerHandler_parseBlobPath(t *testing.T) { wantName string wantDigest string }{ - { - path: "library/nginx/blobs/" + testSHA256Digest, - wantName: "library/nginx", - wantDigest: testSHA256Digest, - }, - { - path: "deep/nested/repo/name/blobs/" + testSHA256Digest, - wantName: "deep/nested/repo/name", - wantDigest: testSHA256Digest, - }, - { - path: "myorg/myrepo/blobs/" + testSHA512Digest, - wantName: "myorg/myrepo", - wantDigest: testSHA512Digest, - }, { path: "library/nginx/blobs/sha256:abc123def456", - wantName: "", - wantDigest: "", + wantName: "library/nginx", + wantDigest: "sha256:abc123def456", }, { - path: "library/nginx/blobs/sha512:" + strings.Repeat("a", 64), - wantName: "", - wantDigest: "", + path: "myorg/myrepo/blobs/sha256:0123456789abcdef", + wantName: "myorg/myrepo", + wantDigest: "sha256:0123456789abcdef", + }, + { + path: "deep/nested/repo/name/blobs/sha256:fedcba9876543210", + wantName: "deep/nested/repo/name", + wantDigest: "sha256:fedcba9876543210", }, { path: "invalid/path", @@ -154,1232 +135,94 @@ func TestContainerHandler_parseTagsListPath(t *testing.T) { } } -func TestContainerHandler_TagsListUsesStaleCacheOnUpstreamFailure(t *testing.T) { - tags := `{"name":"library/nginx","tags":["1.0","latest"]}` - upstreamAvailable := true - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - if r.URL.Path != "/v2/library/nginx/tags/list" { - http.NotFound(w, r) - return - } - if !upstreamAvailable { - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - return - } +func TestContainerHandler_BlobDownload_CachesWithAuth(t *testing.T) { + // Set up a mock auth server that returns a token + authServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - w.Header().Set("ETag", `"tags-etag"`) - _, _ = io.WriteString(w, tags) + _ = json.NewEncoder(w).Encode(map[string]string{"token": "test-token-123"}) })) - defer upstream.Close() + defer authServer.Close() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.MetadataTTL = 0 - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - - first := httptest.NewRecorder() - h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/tags/list?n=2", nil)) - if first.Code != http.StatusOK { - t.Fatalf("initial status = %d, want 200: %s", first.Code, first.Body.String()) - } - if first.Body.String() != tags { - t.Errorf("initial body = %q, want %q", first.Body.String(), tags) + // Set up mock fetcher that captures headers + var capturedHeaders http.Header + mf := &mockFetcherWithHeaders{ + fetchFn: func(_ context.Context, _ string, headers http.Header) (*fetch.Artifact, error) { + capturedHeaders = headers + return &fetch.Artifact{ + Body: io.NopCloser(bytes.NewReader([]byte("blob-content"))), + Size: 12, + ContentType: "application/octet-stream", + }, nil + }, } - upstreamAvailable = false - second := httptest.NewRecorder() - h.Routes().ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/library/nginx/tags/list?n=2", nil)) - if second.Code != http.StatusOK { - t.Fatalf("stale status = %d, want 200: %s", second.Code, second.Body.String()) + dir := t.TempDir() + db, err := database.Create(dir + "/test.db") + if err != nil { + t.Fatalf("failed to create test database: %v", err) } - if second.Body.String() != tags { - t.Errorf("stale body = %q, want %q", second.Body.String(), tags) - } - if got := second.Header().Get("Warning"); got != `110 - "Response is Stale"` { - t.Errorf("Warning = %q, want stale warning", got) - } - if upstreamRequests != 2 { - t.Errorf("upstream requests = %d, want 2", upstreamRequests) - } -} + t.Cleanup(func() { _ = db.Close() }) -func TestContainerHandler_TagsListCachesPaginationLink(t *testing.T) { - tags := `{"name":"library/nginx","tags":["1.0"]}` - upstreamAvailable := true - upstreamRequests := 0 - var upstream *httptest.Server - upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - if !upstreamAvailable { - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Link", `<`+upstream.URL+`/v2/library/nginx/tags/list?last=1.0&n=2>; rel="next"`) - _, _ = io.WriteString(w, tags) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.MetadataTTL = time.Hour - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://proxy.example.test"} - wantLink := `; rel="next"` - - warmRequest := httptest.NewRequest(http.MethodGet, "/library/nginx/tags/list?n=2", nil) - warm := httptest.NewRecorder() - h.Routes().ServeHTTP(warm, warmRequest) - if warm.Code != http.StatusOK { - t.Fatalf("warm status = %d, want 200: %s", warm.Code, warm.Body.String()) + store := newMockStorage() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy := &Proxy{ + DB: db, + Storage: store, + Fetcher: mf, + Logger: logger, + HTTPClient: &http.Client{}, } - if got := warm.Header().Get("Link"); got != wantLink { - t.Errorf("warm Link = %q, want %q", got, wantLink) - } - - fresh := httptest.NewRecorder() - h.Routes().ServeHTTP(fresh, httptest.NewRequest(http.MethodGet, "/library/nginx/tags/list?n=2", nil)) - if fresh.Code != http.StatusOK { - t.Fatalf("fresh status = %d, want 200: %s", fresh.Code, fresh.Body.String()) - } - if got := fresh.Header().Get("Link"); got != wantLink { - t.Errorf("fresh Link = %q, want %q", got, wantLink) - } - if upstreamRequests != 1 { - t.Fatalf("upstream requests after fresh cache hit = %d, want 1", upstreamRequests) - } - - proxy.MetadataTTL = 0 - upstreamAvailable = false - stale := httptest.NewRecorder() - h.Routes().ServeHTTP(stale, httptest.NewRequest(http.MethodGet, "/library/nginx/tags/list?n=2", nil)) - if stale.Code != http.StatusOK { - t.Fatalf("stale status = %d, want 200: %s", stale.Code, stale.Body.String()) - } - if got := stale.Header().Get("Link"); got != wantLink { - t.Errorf("stale Link = %q, want %q", got, wantLink) - } - if got := stale.Header().Get("Warning"); got != `110 - "Response is Stale"` { - t.Errorf("stale Warning = %q, want stale warning", got) - } - if upstreamRequests != 2 { - t.Errorf("upstream requests after stale fallback = %d, want 2", upstreamRequests) - } -} - -func TestContainerHandler_TagsListRewritesRelativePaginationLinkForNamedRegistry(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v2/owner/repo/tags/list" { - http.NotFound(w, r) - return - } - if r.URL.Query().Get("n") != "1" { - http.Error(w, "unexpected page size", http.StatusBadRequest) - return - } - - w.Header().Set("Content-Type", "application/json") - if r.URL.Query().Get("last") == "" { - w.Header().Set("Link", `; rel="next"`) - _, _ = io.WriteString(w, `{"name":"owner/repo","tags":["1.0"]}`) - return - } - if r.URL.Query().Get("last") != "1.0" { - http.Error(w, "unexpected pagination token", http.StatusBadRequest) - return - } - _, _ = io.WriteString(w, `{"name":"owner/repo","tags":["2.0"]}`) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", "", map[string]string{"test": upstream.URL}) - routes := http.StripPrefix("/v2", h.Routes()) - - first := httptest.NewRecorder() - routes.ServeHTTP(first, httptest.NewRequest(http.MethodGet, - "/v2/upstream/test/owner/repo/tags/list?n=1", nil)) - if first.Code != http.StatusOK { - t.Fatalf("first page status = %d, want 200: %s", first.Code, first.Body.String()) - } - const wantLink = `; rel="next"` - if got := first.Header().Get("Link"); got != wantLink { - t.Fatalf("first page Link = %q, want %q", got, wantLink) - } - - nextURL := strings.TrimPrefix(strings.SplitN(first.Header().Get("Link"), ">", 2)[0], "<") - next := httptest.NewRecorder() - routes.ServeHTTP(next, httptest.NewRequest(http.MethodGet, nextURL, nil)) - if next.Code != http.StatusOK { - t.Fatalf("next page status = %d, want 200: %s", next.Code, next.Body.String()) - } - if got, want := next.Body.String(), `{"name":"owner/repo","tags":["2.0"]}`; got != want { - t.Errorf("next page body = %q, want %q", got, want) - } -} - -func TestContainerHandler_NamedOCIRegistryServesHelmArtifacts(t *testing.T) { - const blob = "chart archive" - digest := "sha256:" + sha256Hex(blob) - manifest := `{"schemaVersion":2,"config":{"mediaType":"application/vnd.cncf.helm.config.v1+json"},"layers":[{"mediaType":"application/vnd.cncf.helm.chart.content.v1.tar+gzip","digest":"` + digest + `"}]}` - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/v2/owner/demo/manifests/1.0.0": - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Set("Docker-Content-Digest", "sha256:"+sha256Hex(manifest)) - _, _ = io.WriteString(w, manifest) - case "/v2/owner/demo/blobs/" + digest: - w.Header().Set("Content-Type", "application/vnd.cncf.helm.chart.content.v1.tar+gzip") - _, _ = io.WriteString(w, blob) - default: - http.NotFound(w, r) - } - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", "", map[string]string{"ghcr": upstream.URL}) - - manifestResponse := httptest.NewRecorder() - h.Routes().ServeHTTP(manifestResponse, - httptest.NewRequest(http.MethodGet, "/upstream/ghcr/owner/demo/manifests/1.0.0", nil)) - if manifestResponse.Code != http.StatusOK { - t.Fatalf("manifest status = %d, want 200: %s", manifestResponse.Code, manifestResponse.Body.String()) - } - if got := manifestResponse.Header().Get("Content-Type"); got != "application/vnd.oci.image.manifest.v1+json" { - t.Errorf("manifest Content-Type = %q", got) - } - - blobResponse := httptest.NewRecorder() - h.Routes().ServeHTTP(blobResponse, - httptest.NewRequest(http.MethodGet, "/upstream/ghcr/owner/demo/blobs/"+digest, nil)) - if blobResponse.Code != http.StatusOK { - t.Fatalf("blob status = %d, want 200: %s", blobResponse.Code, blobResponse.Body.String()) - } - if got := blobResponse.Header().Get("Content-Type"); got != "application/vnd.cncf.helm.chart.content.v1.tar+gzip" { - t.Errorf("blob Content-Type = %q", got) - } -} - -func TestContainerHandler_registryURLForUsesLongestRepositoryPrefix(t *testing.T) { - h := &ContainerHandler{registryURL: "https://registry-1.docker.io"} - h.RegisterRegistry("homebrew", "https://example.test") - h.RegisterRegistry("homebrew/core", "https://ghcr.io/") - - tests := map[string]string{ - "homebrew/core": "https://ghcr.io", - "homebrew/core/jq": "https://ghcr.io", - "homebrew/portable-ruby": "https://example.test", - "homebrew-core/jq": "https://registry-1.docker.io", - "library/homebrew/core/jq": "https://registry-1.docker.io", - } - for name, want := range tests { - if got := h.registryURLFor(name); got != want { - t.Errorf("registryURLFor(%q) = %q, want %q", name, got, want) - } - } -} - -func TestContainerHandler_BlobDownload_DiscoversBearerChallenge(t *testing.T) { - blob := "upstream blob" - digest := sha256Digest([]byte(blob)) - registryRequests := 0 - tokenRequests := 0 - var upstream *httptest.Server - upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/token": - tokenRequests++ - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]any{ - "token": "discovered-token", - "expires_in": 3600, - }) - case "/v2/library/nginx/blobs/" + digest: - registryRequests++ - if r.Header.Get("Authorization") != "Bearer discovered-token" { - w.Header().Set("WWW-Authenticate", `Bearer realm="`+upstream.URL+`/token",service="registry.test",scope="repository:library/nginx:pull"`) - http.Error(w, "authentication required", http.StatusUnauthorized) - return - } - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = io.WriteString(w, blob) - default: - http.NotFound(w, r) - } - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - authTransport := upstreamhttp.NewTransport(http.DefaultTransport, nil) - client := &http.Client{Transport: authTransport} - artifactFetcher := fetch.NewFetcher( - fetch.WithHTTPClient(client), - fetch.WithMaxRetries(0), - ) - t.Cleanup(func() { _ = artifactFetcher.Close() }) - proxy.Fetcher = artifactFetcher - proxy.HTTPClient = client h := &ContainerHandler{ proxy: proxy, - registryURL: upstream.URL, + registryURL: "https://registry-1.docker.io", + authURL: authServer.URL, proxyURL: "http://localhost:8080", } - for range 2 { - req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if got := w.Body.String(); got != blob { - t.Errorf("body = %q, want %q", got, blob) - } - } - - if tokenRequests != 1 { - t.Errorf("token requests = %d, want 1", tokenRequests) - } - if registryRequests != 2 { - t.Errorf("registry requests = %d, want 2", registryRequests) - } -} - -func TestContainerHandler_HomebrewBlobDoesNotForwardClientCredentialsToRegistryOrCDN(t *testing.T) { - blob := "homebrew bottle" - digest := sha256Digest([]byte(blob)) - var registryAuthorization, cdnAuthorization string - - cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - cdnAuthorization = r.Header.Get("Authorization") - w.Header().Set("Content-Type", "application/vnd.homebrew.bottle") - _, _ = io.WriteString(w, blob) - })) - defer cdn.Close() - - registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - registryAuthorization = r.Header.Get("Authorization") - http.Redirect(w, r, cdn.URL+"/bottle", http.StatusTemporaryRedirect) - })) - defer registry.Close() - - defaultRequests := 0 - defaultRegistry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defaultRequests++ - http.NotFound(w, r) - })) - defer defaultRegistry.Close() - - proxy, _, _, _ := setupTestProxy(t) - client := registry.Client() - artifactFetcher := fetch.NewFetcher( - fetch.WithHTTPClient(client), - fetch.WithMaxRetries(0), - ) - t.Cleanup(func() { _ = artifactFetcher.Close() }) - proxy.Fetcher = artifactFetcher - - h := &ContainerHandler{proxy: proxy, registryURL: defaultRegistry.URL} - h.RegisterRegistry("homebrew/core", registry.URL) - req := httptest.NewRequest(http.MethodGet, "/homebrew/core/jq/blobs/"+digest, nil) - req.Header.Set("Authorization", "Bearer client-secret") + handler := h.Routes() + req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd", nil) w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + handler.ServeHTTP(w, req) if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + t.Errorf("got status %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) } - if got := w.Body.String(); got != blob { - t.Errorf("body = %q, want %q", got, blob) + + // Verify auth header was passed to the fetcher + if capturedHeaders == nil { + t.Fatal("expected headers to be passed to fetcher, got nil") } - if got := w.Header().Get("Content-Type"); got != "application/vnd.homebrew.bottle" { - t.Errorf("Content-Type = %q, want application/vnd.homebrew.bottle", got) + auth := capturedHeaders.Get("Authorization") + if auth != "Bearer test-token-123" { + t.Errorf("Authorization = %q, want %q", auth, "Bearer test-token-123") } - if registryAuthorization != "" { - t.Errorf("registry Authorization = %q, want empty", registryAuthorization) - } - if cdnAuthorization != "" { - t.Errorf("CDN Authorization = %q, want empty", cdnAuthorization) - } - if defaultRequests != 0 { - t.Errorf("default registry requests = %d, want 0", defaultRequests) + + // Verify response headers + if got := w.Header().Get("Docker-Content-Digest"); got != "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" { + t.Errorf("Docker-Content-Digest = %q, want digest", got) } } -func TestContainerHandler_BlobDigestMismatchIsNotCached(t *testing.T) { - proxy, _, store, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("wrong bottle")), - ContentType: "application/octet-stream", - } - digest := sha256Digest([]byte("expected bottle")) - h := &ContainerHandler{proxy: proxy, registryURL: "https://registry.example.test"} - - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/homebrew/core/jq/blobs/"+digest, nil)) - - if w.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusBadGateway, w.Body.String()) - } - if !strings.Contains(w.Body.String(), "DIGEST_INVALID") { - t.Errorf("body = %q, want DIGEST_INVALID", w.Body.String()) - } - cached, err := proxy.GetCachedArtifact(t.Context(), "oci", "homebrew/core/jq", digest, digest) - if err != nil { - t.Fatalf("checking cache: %v", err) - } - if cached != nil { - t.Error("digest-mismatched blob was recorded in the cache") - } - if len(store.files) != 0 { - t.Errorf("stored files = %d, want 0", len(store.files)) - } +// mockFetcherWithHeaders captures headers passed to FetchWithHeaders. +type mockFetcherWithHeaders struct { + fetchFn func(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error) } -func TestContainerHandler_CachedImagePullSurvivesRegistryAndTokenOutages(t *testing.T) { - manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` - blob := "cached image blob" - manifestDigest := sha256Digest([]byte(manifest)) - blobDigest := sha256Digest([]byte(blob)) - registryAvailable := true - tokenAvailable := true - registryRequests := 0 - tokenRequests := 0 - - tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - tokenRequests++ - if !tokenAvailable { - http.Error(w, "token service unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]any{ - "token": "discovered-token", - "expires_in": 3600, - }) - })) - defer tokenServer.Close() - - registryServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - registryRequests++ - if !registryAvailable { - http.Error(w, "registry unavailable", http.StatusServiceUnavailable) - return - } - if r.Header.Get("Authorization") != "Bearer discovered-token" { - w.Header().Set("WWW-Authenticate", `Bearer realm="`+tokenServer.URL+`",service="registry.test",scope="repository:library/nginx:pull"`) - http.Error(w, "authentication required", http.StatusUnauthorized) - return - } - - switch r.URL.Path { - case "/v2/library/nginx/manifests/latest": - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Set("Docker-Content-Digest", manifestDigest) - _, _ = io.WriteString(w, manifest) - case "/v2/library/nginx/blobs/" + blobDigest: - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = io.WriteString(w, blob) - default: - http.NotFound(w, r) - } - })) - defer registryServer.Close() - - warmProxy, db, store, _ := setupTestProxy(t) - warmClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, nil)} - warmFetcher := fetch.NewFetcher( - fetch.WithHTTPClient(warmClient), - fetch.WithMaxRetries(0), - ) - t.Cleanup(func() { _ = warmFetcher.Close() }) - warmProxy.Fetcher = warmFetcher - warmProxy.HTTPClient = warmClient - warmProxy.MetadataTTL = time.Hour - warmHandler := (&ContainerHandler{ - proxy: warmProxy, - registryURL: registryServer.URL, - proxyURL: "http://localhost:8080", - }).Routes() - - for _, request := range []struct { - path string - body string - }{ - {path: "/library/nginx/manifests/latest", body: manifest}, - {path: "/library/nginx/blobs/" + blobDigest, body: blob}, - } { - response := httptest.NewRecorder() - warmHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, request.path, nil)) - if response.Code != http.StatusOK { - t.Fatalf("warming %s: status = %d, want %d; body: %s", request.path, response.Code, http.StatusOK, response.Body.String()) - } - if got := response.Body.String(); got != request.body { - t.Fatalf("warming %s: body = %q, want %q", request.path, got, request.body) - } - } - - warmRegistryRequests := registryRequests - warmTokenRequests := tokenRequests - registryAvailable = false - tokenAvailable = false - - offlineClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, nil)} - offlineFetcher := fetch.NewFetcher( - fetch.WithHTTPClient(offlineClient), - fetch.WithMaxRetries(0), - ) - t.Cleanup(func() { _ = offlineFetcher.Close() }) - offlineProxy := NewProxy(db, store, offlineFetcher, fetch.NewResolver(), warmProxy.Logger) - offlineProxy.HTTPClient = offlineClient - offlineProxy.MetadataTTL = time.Hour - offlineHandler := (&ContainerHandler{ - proxy: offlineProxy, - registryURL: registryServer.URL, - proxyURL: "http://localhost:8080", - }).Routes() - - for _, request := range []struct { - name string - path string - body string - digest string - }{ - {name: "tag manifest", path: "/library/nginx/manifests/latest", body: manifest, digest: manifestDigest}, - {name: "digest manifest", path: "/library/nginx/manifests/" + manifestDigest, body: manifest, digest: manifestDigest}, - {name: "blob", path: "/library/nginx/blobs/" + blobDigest, body: blob, digest: blobDigest}, - } { - t.Run(request.name, func(t *testing.T) { - response := httptest.NewRecorder() - offlineHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, request.path, nil)) - if response.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", response.Code, http.StatusOK, response.Body.String()) - } - if got := response.Body.String(); got != request.body { - t.Errorf("body = %q, want %q", got, request.body) - } - if got := response.Header().Get("Docker-Content-Digest"); got != request.digest { - t.Errorf("Docker-Content-Digest = %q, want %q", got, request.digest) - } - }) - } - - if registryRequests != warmRegistryRequests { - t.Errorf("offline registry requests = %d, want 0", registryRequests-warmRegistryRequests) - } - if tokenRequests != warmTokenRequests { - t.Errorf("offline token requests = %d, want 0", tokenRequests-warmTokenRequests) - } +func (f *mockFetcherWithHeaders) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { + return f.FetchWithHeaders(ctx, url, nil) } -func TestContainerHandler_BlobDownload_CacheHitSkipsAuth(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" - seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") - store.seekable = true - - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - upstreamRequests++ - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - })) - defer upstream.Close() - - h := &ContainerHandler{ - proxy: proxy, - registryURL: upstream.URL, - proxyURL: "http://localhost:8080", - } - - req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil) - req.Header.Set("Range", "bytes=0-5") - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusPartialContent { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusPartialContent, w.Body.String()) - } - if got := w.Body.String(); got != "cached" { - t.Errorf("body = %q, want %q", got, "cached") - } - if got := w.Header().Get("Content-Range"); got != "bytes 0-5/11" { - t.Errorf("Content-Range = %q, want %q", got, "bytes 0-5/11") - } - if upstreamRequests != 0 { - t.Errorf("upstream requests = %d, want 0", upstreamRequests) - } - if fetcher.fetchCalled { - t.Error("fetcher should not be called on cache hit") - } +func (f *mockFetcherWithHeaders) FetchWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error) { + return f.fetchFn(ctx, url, headers) } -func TestContainerHandler_BlobHead_CacheHitSkipsUpstreamAndAuth(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" - seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") - - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - upstreamRequests++ - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - })) - defer upstream.Close() - proxy.HTTPClient = upstream.Client() - - h := &ContainerHandler{ - proxy: proxy, - registryURL: upstream.URL, - proxyURL: "http://localhost:8080", - } - - req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if got := w.Header().Get("Docker-Content-Digest"); got != digest { - t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) - } - if got := w.Header().Get("Content-Length"); got != "11" { - t.Errorf("Content-Length = %q, want %q", got, "11") - } - if w.Body.Len() != 0 { - t.Errorf("HEAD response body length = %d, want 0", w.Body.Len()) - } - if upstreamRequests != 0 { - t.Errorf("upstream requests = %d, want 0", upstreamRequests) - } - if fetcher.fetchCalled { - t.Error("fetcher should not be called on cache hit") - } -} - -func TestContainerHandler_BlobHead_DirectServeRedirects(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" - seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") - store.signedURL = "https://storage.example.test/cached-blob?signature=test" - proxy.DirectServe = true - - h := &ContainerHandler{ - proxy: proxy, - registryURL: "https://registry.example.test", - proxyURL: "http://localhost:8080", - } - - req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusFound { - t.Fatalf("status = %d, want %d", w.Code, http.StatusFound) - } - if got := w.Header().Get("Location"); got != store.signedURL { - t.Errorf("Location = %q, want %q", got, store.signedURL) - } - wantETag := `"` + sha256Hex("cached blob") + `"` - if got := w.Header().Get("ETag"); got != wantETag { - t.Errorf("ETag = %q, want %q", got, wantETag) - } - if w.Body.Len() != 0 { - t.Errorf("HEAD response body length = %d, want 0", w.Body.Len()) - } - if fetcher.fetchCalled { - t.Error("fetcher should not be called on cache hit") - } -} - -func TestContainerHandler_ManifestByDigest_CacheHitSkipsUpstream(t *testing.T) { - manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` - digest := sha256Digest([]byte(manifest)) - lastModified := time.Date(2026, time.August, 14, 9, 30, 0, 0, time.UTC) - upstreamAvailable := true - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - if !upstreamAvailable { - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - return - } - if r.URL.Path != "/v2/library/nginx/manifests/"+digest { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Set("Docker-Content-Digest", digest) - w.Header().Set("ETag", `"manifest-etag"`) - w.Header().Set("Last-Modified", lastModified.Format(http.TimeFormat)) - if r.Method != http.MethodHead { - _, _ = io.WriteString(w, manifest) - } - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - - first := httptest.NewRecorder() - h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil)) - if first.Code != http.StatusOK { - t.Fatalf("initial status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String()) - } - if first.Body.String() != manifest { - t.Fatalf("initial body = %q, want %q", first.Body.String(), manifest) - } - - upstreamAvailable = false - second := httptest.NewRecorder() - h.Routes().ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil)) - if second.Code != http.StatusOK { - t.Fatalf("cached status = %d, want %d; body: %s", second.Code, http.StatusOK, second.Body.String()) - } - if second.Body.String() != manifest { - t.Errorf("cached body = %q, want %q", second.Body.String(), manifest) - } - if got := second.Header().Get("Docker-Content-Digest"); got != digest { - t.Errorf("cached Docker-Content-Digest = %q, want %q", got, digest) - } - if got := second.Header().Get("Last-Modified"); got != lastModified.Format(http.TimeFormat) { - t.Errorf("cached Last-Modified = %q, want %q", got, lastModified.Format(http.TimeFormat)) - } - - conditionalRequest := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil) - conditionalRequest.Header.Set("If-None-Match", `"manifest-etag"`) - conditional := httptest.NewRecorder() - h.Routes().ServeHTTP(conditional, conditionalRequest) - if conditional.Code != http.StatusNotModified { - t.Fatalf("conditional status = %d, want %d", conditional.Code, http.StatusNotModified) - } - if got := conditional.Header().Get("ETag"); got != `"manifest-etag"` { - t.Errorf("conditional ETag = %q, want %q", got, `"manifest-etag"`) - } - if conditional.Body.Len() != 0 { - t.Errorf("conditional body length = %d, want 0", conditional.Body.Len()) - } - - head := httptest.NewRecorder() - h.Routes().ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/library/nginx/manifests/"+digest, nil)) - if head.Code != http.StatusOK { - t.Fatalf("cached HEAD status = %d, want %d", head.Code, http.StatusOK) - } - wantLength := strconv.Itoa(len(manifest)) - if got := head.Header().Get("Content-Length"); got != wantLength { - t.Errorf("cached HEAD Content-Length = %q, want %q", got, wantLength) - } - if head.Body.Len() != 0 { - t.Errorf("cached HEAD body length = %d, want 0", head.Body.Len()) - } - if upstreamRequests != 1 { - t.Errorf("upstream requests = %d, want 1", upstreamRequests) - } -} - -func TestContainerHandler_ManifestDigestMismatchIsNotCached(t *testing.T) { - manifest := `{"schemaVersion":2}` - digest := sha256Digest([]byte("different manifest")) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Set("Docker-Content-Digest", digest) - _, _ = io.WriteString(w, manifest) - })) - defer upstream.Close() - - proxy, db, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL} - req := httptest.NewRequest(http.MethodGet, "/homebrew/core/jq/manifests/"+digest, nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusBadGateway, w.Body.String()) - } - if !strings.Contains(w.Body.String(), "DIGEST_INVALID") { - t.Errorf("body = %q, want DIGEST_INVALID", w.Body.String()) - } - cacheKey := h.containerManifestSharedCacheKey(upstream.URL, "homebrew/core/jq", digest, normalizeContainerManifestAccept(containerManifestAccept(req))) - entry, err := db.GetMetadataCache(containerManifestCacheEcosystem, cacheKey) - if err != nil { - t.Fatalf("checking manifest cache: %v", err) - } - if entry != nil { - t.Error("digest-mismatched manifest was recorded in the cache") - } -} - -// TestContainerHandler_ManifestDigestsAreVerified checks that a sha512 -// digest, in the reference or in Docker-Content-Digest, is verified like a -// sha256 one, and that a digest the proxy can't compute is refused rather -// than cached unverified. -func TestContainerHandler_ManifestDigestsAreVerified(t *testing.T) { - manifest := `{"schemaVersion":2}` - sum := sha512.Sum512([]byte(manifest)) - sha512Digest := "sha512:" + hex.EncodeToString(sum[:]) - otherSHA512 := "sha512:" + strings.Repeat("b", 128) - unsupported := "sha1:" + strings.Repeat("c", 40) - - tests := []struct { - name, reference, header string - wantOK bool - }{ - {"sha512 reference", sha512Digest, sha512Digest, true}, - {"sha512 reference without header", sha512Digest, "", true}, - {"tag with sha512 header", "latest", sha512Digest, true}, - {"mismatched sha512 reference", otherSHA512, "", false}, - {"tag with mismatched sha512 header", "latest", otherSHA512, false}, - {"unsupported reference", unsupported, "", false}, - {"tag with unsupported header", "latest", unsupported, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - if tt.header != "" { - w.Header().Set("Docker-Content-Digest", tt.header) - } - _, _ = io.WriteString(w, manifest) - })) - defer upstream.Close() - - proxy, db, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL} - req := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+tt.reference, nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if !tt.wantOK { - if w.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusBadGateway, w.Body.String()) - } - requireManifestNotCached(t, h, db, upstream.URL, req, tt.reference, tt.header) - return - } - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if got := w.Body.String(); got != manifest { - t.Errorf("body = %q, want %q", got, manifest) - } - }) - } -} - -func requireManifestNotCached(t *testing.T, h *ContainerHandler, db *database.DB, upstreamURL string, req *http.Request, keys ...string) { - t.Helper() - accept := normalizeContainerManifestAccept(containerManifestAccept(req)) - for _, key := range keys { - if key == "" { - continue - } - entry, err := db.GetMetadataCache(containerManifestCacheEcosystem, - h.containerManifestSharedCacheKey(upstreamURL, "library/nginx", key, accept)) - if err != nil { - t.Fatalf("checking manifest cache: %v", err) - } - if entry != nil { - t.Errorf("manifest was cached under unverified %s", key) - } - } -} - -func TestContainerHandler_ManifestTagWithInvalidDigestIsNotAliased(t *testing.T) { - manifest := `{"schemaVersion":2}` - invalidDigest := sha256Digest([]byte("different manifest")) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Set("Docker-Content-Digest", invalidDigest) - _, _ = io.WriteString(w, manifest) - })) - defer upstream.Close() - - proxy, db, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL} - req := httptest.NewRequest(http.MethodGet, "/homebrew/core/jq/manifests/latest", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusBadGateway, w.Body.String()) - } - cacheKey := h.containerManifestSharedCacheKey(upstream.URL, "homebrew/core/jq", invalidDigest, normalizeContainerManifestAccept(containerManifestAccept(req))) - entry, err := db.GetMetadataCache(containerManifestCacheEcosystem, cacheKey) - if err != nil { - t.Fatalf("checking manifest cache: %v", err) - } - if entry != nil { - t.Error("tag manifest was cached under an unverified digest") - } -} - -func TestContainerHandler_ManifestByTag_UsesStaleCacheOnUpstreamFailure(t *testing.T) { - manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json"}` - digest := sha256Digest([]byte(manifest)) - upstreamAvailable := true - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - upstreamRequests++ - if !upstreamAvailable { - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json") - w.Header().Set("Docker-Content-Digest", digest) - _, _ = io.WriteString(w, manifest) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.MetadataTTL = 0 - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - - first := httptest.NewRecorder() - h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) - if first.Code != http.StatusOK { - t.Fatalf("initial status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String()) - } - - upstreamAvailable = false - second := httptest.NewRecorder() - h.Routes().ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) - if second.Code != http.StatusOK { - t.Fatalf("stale status = %d, want %d; body: %s", second.Code, http.StatusOK, second.Body.String()) - } - if second.Body.String() != manifest { - t.Errorf("stale body = %q, want %q", second.Body.String(), manifest) - } - if got := second.Header().Get("Warning"); got != `110 - "Response is Stale"` { - t.Errorf("Warning = %q, want stale warning", got) - } - if got := second.Header().Get("Docker-Content-Digest"); got != digest { - t.Errorf("stale Docker-Content-Digest = %q, want %q", got, digest) - } - if upstreamRequests != 2 { - t.Errorf("upstream requests = %d, want 2", upstreamRequests) - } -} - -func TestContainerHandler_ManifestVariantCacheNormalizesCompatibleAccept(t *testing.T) { - manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json"}` - digest := sha256Digest([]byte(manifest)) - upstreamAvailable := true - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - if !upstreamAvailable { - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", "") - w.Header().Set("Docker-Content-Digest", digest) - _, _ = io.WriteString(w, manifest) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.MetadataTTL = 0 - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - - firstRequest := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil) - firstRequest.Header.Set("Accept", "application/vnd.oci.image.index.v1+json;q=1, application/vnd.oci.image.manifest.v1+json;q=1, application/vnd.oci.image.index.v1+json;q=1") - first := httptest.NewRecorder() - h.Routes().ServeHTTP(first, firstRequest) - if first.Code != http.StatusOK { - t.Fatalf("initial status = %d, want 200: %s", first.Code, first.Body.String()) - } - - upstreamAvailable = false - secondRequest := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil) - secondRequest.Header.Set("Accept", " application/vnd.oci.image.manifest.v1+json , application/vnd.oci.image.index.v1+json ") - second := httptest.NewRecorder() - h.Routes().ServeHTTP(second, secondRequest) - if second.Code != http.StatusOK { - t.Fatalf("stale status = %d, want 200: %s", second.Code, second.Body.String()) - } - if second.Body.String() != manifest { - t.Errorf("stale body = %q, want %q", second.Body.String(), manifest) - } - if got := second.Header().Get("Warning"); got != `110 - "Response is Stale"` { - t.Errorf("Warning = %q, want stale warning", got) - } - if upstreamRequests != 2 { - t.Errorf("upstream requests = %d, want 2", upstreamRequests) - } -} - -func TestContainerHandler_ManifestMigratesLegacyAcceptCacheKey(t *testing.T) { - digest := "sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd" - manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json"}` - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - http.Error(w, "upstream should not be called", http.StatusServiceUnavailable) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.MetadataTTL = time.Hour - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - accept := "application/vnd.oci.image.index.v1+json;q=1, application/vnd.oci.image.manifest.v1+json;q=1" - cacheAccept := normalizeContainerManifestAccept(accept) - legacyCacheKey := h.containerManifestCacheKey(upstream.URL, "library/nginx", "latest", accept) - cacheKey := h.containerManifestSharedCacheKey(upstream.URL, "library/nginx", "latest", cacheAccept) - if legacyCacheKey == cacheKey { - t.Fatal("legacy and shared cache keys are equal") - } - - request := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil) - request.Header.Set("Accept", accept) - legacyManifest := &cachedContainerManifest{ - body: []byte(manifest), - contentType: "application/vnd.oci.image.index.v1+json", - contentDigest: digest, - fetchedAt: time.Now(), - } - if err := h.storeContainerManifest(request.Context(), legacyCacheKey, legacyManifest); err != nil { - t.Fatalf("store legacy manifest: %v", err) - } - - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, request) - if response.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", response.Code, response.Body.String()) - } - if response.Body.String() != manifest { - t.Errorf("body = %q, want %q", response.Body.String(), manifest) - } - if upstreamRequests != 0 { - t.Errorf("upstream requests = %d, want 0", upstreamRequests) - } - migrated, err := h.loadContainerManifest(request.Context(), cacheKey) - if err != nil { - t.Fatalf("load migrated manifest: %v", err) - } - if migrated == nil { - t.Error("shared cache entry was not created") - } -} - -func TestContainerHandler_ManifestDualWritesLegacyAcceptCacheKeys(t *testing.T) { - manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` - digest := sha256Digest([]byte(manifest)) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v2/library/nginx/manifests/latest" { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Set("Docker-Content-Digest", digest) - _, _ = io.WriteString(w, manifest) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - accept := "application/vnd.oci.image.manifest.v1+json;q=1, application/vnd.oci.image.manifest.v1+json;q=1" - request := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil) - request.Header.Set("Accept", accept) - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, request) - if response.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", response.Code, response.Body.String()) - } - - for _, reference := range []string{"latest", digest} { - legacyCacheKey := h.containerManifestCacheKey(upstream.URL, "library/nginx", reference, accept) - cached, err := h.loadContainerManifest(request.Context(), legacyCacheKey) - if err != nil { - t.Fatalf("load legacy %s manifest: %v", reference, err) - } - if cached == nil { - t.Errorf("legacy %s cache entry was not written", reference) - } - } -} - -func TestContainerHandler_ManifestVariantCacheHonorsSpecificAcceptExclusions(t *testing.T) { - manifest := `{"schemaVersion":2}` - digest := sha256Digest([]byte(manifest)) - upstreamAvailable := true - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if !upstreamAvailable { - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json") - w.Header().Set("Docker-Content-Digest", digest) - _, _ = io.WriteString(w, manifest) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.MetadataTTL = 0 - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - - warmRequest := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil) - warmRequest.Header.Set("Accept", "application/vnd.oci.image.index.v1+json;q=0, */*;q=1") - warm := httptest.NewRecorder() - h.Routes().ServeHTTP(warm, warmRequest) - if warm.Code != http.StatusOK { - t.Fatalf("warm status = %d, want 200", warm.Code) - } - - upstreamAvailable = false - offlineRequest := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil) - offlineRequest.Header.Set("Accept", "application/vnd.oci.image.index.v1+json;q=0, */*;q=1") - offline := httptest.NewRecorder() - h.Routes().ServeHTTP(offline, offlineRequest) - if offline.Code != http.StatusServiceUnavailable { - t.Errorf("offline status = %d, want 503", offline.Code) - } -} - -func TestContainerHandler_ManifestVariantCacheHonorsParameterizedAcceptExclusions(t *testing.T) { - contentType := "application/vnd.oci.image.index.v1+json; charset=utf-8" - upstreamAvailable := true - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if !upstreamAvailable { - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", contentType) - _, _ = io.WriteString(w, `{"schemaVersion":2}`) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.MetadataTTL = 0 - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - accept := "application/vnd.oci.image.index.v1+json;q=1, application/vnd.oci.image.index.v1+json;charset=utf-8;q=0" - - warmRequest := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil) - warmRequest.Header.Set("Accept", accept) - warm := httptest.NewRecorder() - h.Routes().ServeHTTP(warm, warmRequest) - if warm.Code != http.StatusOK { - t.Fatalf("warm status = %d, want 200", warm.Code) - } - - upstreamAvailable = false - offlineRequest := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil) - offlineRequest.Header.Set("Accept", accept) - offline := httptest.NewRecorder() - h.Routes().ServeHTTP(offline, offlineRequest) - if offline.Code != http.StatusServiceUnavailable { - t.Errorf("offline status = %d, want 503", offline.Code) - } -} - -func TestContainerHandler_ManifestByTag_CachesDigestAlias(t *testing.T) { - manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` - digest := sha256Digest([]byte(manifest)) - upstreamAvailable := true - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - if !upstreamAvailable { - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - return - } - if r.URL.Path != "/v2/library/nginx/manifests/latest" { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Set("Docker-Content-Digest", digest) - _, _ = io.WriteString(w, manifest) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - - first := httptest.NewRecorder() - h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) - if first.Code != http.StatusOK { - t.Fatalf("tag status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String()) - } - - upstreamAvailable = false - byDigest := httptest.NewRecorder() - h.Routes().ServeHTTP(byDigest, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil)) - if byDigest.Code != http.StatusOK { - t.Fatalf("digest status = %d, want %d; body: %s", byDigest.Code, http.StatusOK, byDigest.Body.String()) - } - if byDigest.Body.String() != manifest { - t.Errorf("digest body = %q, want %q", byDigest.Body.String(), manifest) - } - if got := byDigest.Header().Get("Docker-Content-Digest"); got != digest { - t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) - } - if upstreamRequests != 1 { - t.Errorf("upstream requests = %d, want 1", upstreamRequests) - } -} - -func TestContainerHandler_ManifestByTag_StaleHeadChecksUpstream(t *testing.T) { - manifest := `{"schemaVersion":2}` - oldDigest := sha256Digest([]byte(manifest)) - newDigest := "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - currentDigest := oldDigest - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Set("Docker-Content-Digest", currentDigest) - w.Header().Set("ETag", `"`+currentDigest+`"`) - if r.Method != http.MethodHead { - _, _ = io.WriteString(w, manifest) - } - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.MetadataTTL = 0 - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - - first := httptest.NewRecorder() - h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) - if first.Code != http.StatusOK { - t.Fatalf("initial status = %d, want %d", first.Code, http.StatusOK) - } - - currentDigest = newDigest - head := httptest.NewRecorder() - h.Routes().ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/library/nginx/manifests/latest", nil)) - if head.Code != http.StatusOK { - t.Fatalf("HEAD status = %d, want %d", head.Code, http.StatusOK) - } - if got := head.Header().Get("Docker-Content-Digest"); got != newDigest { - t.Errorf("Docker-Content-Digest = %q, want %q", got, newDigest) - } - if upstreamRequests != 2 { - t.Errorf("upstream requests = %d, want 2", upstreamRequests) - } +func (f *mockFetcherWithHeaders) Head(_ context.Context, _ string) (int64, string, error) { + return 0, "", nil } func TestContainerHandler_Routes_VersionCheck(t *testing.T) { - h := NewContainerHandlerWithRegistry(nil, "http://localhost:8080", "") + h := NewContainerHandler(nil, "http://localhost:8080") handler := h.Routes() if handler == nil { @@ -1399,144 +242,3 @@ func TestContainerHandler_Routes_VersionCheck(t *testing.T) { t.Errorf("Docker-Distribution-Api-Version = %q, want %q", got, "registry/2.0") } } - -func TestOCIRetentionKey(t *testing.T) { - tests := []struct { - key string - want string - ok bool - }{ - {"pkg:oci/library/nginx", "pkg:oci/library%2Fnginx", true}, - {"pkg:oci/library%2Fnginx", "pkg:oci/library%2Fnginx", true}, - {"pkg:oci/nginx", "pkg:oci/nginx", true}, - {"pkg:oci/upstream/ghcr/owner/app", "pkg:oci/upstream%2Fghcr%2Fowner%2Fapp", true}, - {"pkg:oci/homebrew/core/wget", "pkg:oci/homebrew%2Fcore%2Fwget", true}, - {"pkg:oci/upstream/ghcr", "", false}, - {"pkg:oci/library%2F%2Fnginx", "", false}, - {"pkg:oci/%2Fnginx", "", false}, - {"pkg:oci/library/nginx:latest", "", false}, - {"pkg:oci/homebrew/core/openssl%403", "", false}, - {"pkg:oci/homebrew/core/openssl/3", "pkg:oci/homebrew%2Fcore%2Fopenssl%2F3", true}, - {"pkg:npm/library/nginx", "", false}, - } - for _, tt := range tests { - p, err := purl.Parse(tt.key) - if err != nil { - t.Fatalf("Parse(%s): %v", tt.key, err) - } - got, ok := ociRetentionKey(p) - if ok != tt.ok || (ok && got != tt.want) { - t.Errorf("ociRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) - } - } - if _, ok := ociRetentionKey(nil); ok { - t.Error("ociRetentionKey(nil) accepted") - } -} - -// Docker sends one Accept header per media type. All of them count, both for -// the request upstream and for which cached representation can be served. -func TestContainerHandler_ManifestHonoursSeparateAcceptHeaders(t *testing.T) { - manifest := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json"}` - digest := sha256Digest([]byte(manifest)) - dockerAccept := []string{ - "application/vnd.docker.distribution.manifest.list.v2+json", - "application/vnd.oci.image.index.v1+json", - "application/vnd.oci.image.manifest.v1+json", - "application/vnd.docker.distribution.manifest.v2+json", - } - upstreamAvailable := true - var upstreamAccept string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !upstreamAvailable { - http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) - return - } - upstreamAccept = strings.Join(r.Header.Values("Accept"), ", ") - w.Header().Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json") - w.Header().Set("Docker-Content-Digest", digest) - _, _ = io.WriteString(w, manifest) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.MetadataTTL = 0 - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} - pull := func() *httptest.ResponseRecorder { - req := httptest.NewRequest(http.MethodGet, "/library/busybox/manifests/1", nil) - for _, mediaType := range dockerAccept { - req.Header.Add("Accept", mediaType) - } - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - return w - } - - if w := pull(); w.Code != http.StatusOK { - t.Fatalf("online status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - for _, mediaType := range dockerAccept { - if !strings.Contains(upstreamAccept, mediaType) { - t.Errorf("upstream Accept = %q, missing %s", upstreamAccept, mediaType) - } - } - - upstreamAvailable = false - w := pull() - if w.Code != http.StatusOK { - t.Fatalf("offline status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if w.Body.String() != manifest { - t.Errorf("offline body = %q, want the cached manifest", w.Body.String()) - } -} - -func TestContainerHandler_BlobSHA512(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - const blob = "sha512 layer" - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader(blob)), - ContentType: "application/octet-stream", - } - sum := sha512.Sum512([]byte(blob)) - digest := "sha512:" + hex.EncodeToString(sum[:]) - h := &ContainerHandler{proxy: proxy, registryURL: "https://registry.example.test"} - - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/library/app/blobs/"+digest, nil)) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if w.Body.String() != blob { - t.Errorf("body = %q, want %q", w.Body.String(), blob) - } - if got := w.Header().Get("Docker-Content-Digest"); got != digest { - t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) - } -} - -func TestContainerHandler_HeadManifestByDigestChecksUpstreamDigest(t *testing.T) { - requested := sha256Digest([]byte("requested")) - other := sha256Digest([]byte("other")) - registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", ociManifestMediaType) - w.Header().Set("Docker-Content-Digest", other) - w.Header().Set("Content-Length", "5") - })) - defer registry.Close() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = registry.Client() - h := &ContainerHandler{proxy: proxy, registryURL: registry.URL} - - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/library/app/manifests/"+requested, nil)) - - if w.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want %d", w.Code, http.StatusBadGateway) - } - if got := w.Header().Get("Docker-Content-Digest"); got == other { - t.Errorf("Docker-Content-Digest = %q, the upstream's other digest", got) - } -} diff --git a/internal/handler/cooldown_expiry.go b/internal/handler/cooldown_expiry.go deleted file mode 100644 index c31ace8..0000000 --- a/internal/handler/cooldown_expiry.go +++ /dev/null @@ -1,167 +0,0 @@ -package handler - -import ( - "sync" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/proxy/internal/packageurl" -) - -// A rewrite made with cooldown on stays right until a version it held back -// leaves cooldown: versions it let through stay allowed as time passes, and -// new versions upstream change the document and so the cache key. So the -// cached rewrite expires when the earliest version it withheld becomes -// available, instead of after a fixed time. -// -// Handlers ask cooldown through Proxy.cooldown, which tells every rewrite -// running at that moment when a withheld version becomes available. A -// rewrite may hear of versions another rewrite withheld at the same time; -// that only makes it expire earlier than it has to, never later. - -// maxCooldownRewriteTTL is the longest a rewrite made with cooldown on is -// served, even when it held nothing back. -const maxCooldownRewriteTTL = 24 * time.Hour - -// minCooldownRewriteTTL is the shortest cap on a rewrite made with cooldown -// on, however short the cooldown. -const minCooldownRewriteTTL = time.Minute - -// cooldownRewriteTTL returns the longest a rewrite made with cooldown on is -// served: the shortest cooldown configured, between a minute and a day. It -// only matters when a rewrite held nothing back, or held back a version -// whose availability is unknown. -func (p *Proxy) cooldownRewriteTTL() time.Duration { - ttl := p.ShortestCooldown - switch { - case ttl <= 0 || ttl > maxCooldownRewriteTTL: - return maxCooldownRewriteTTL - case ttl < minCooldownRewriteTTL: - return minCooldownRewriteTTL - } - return ttl -} - -// cooldownWatch holds the rewrites running with cooldown on. -type cooldownWatch struct { - mu sync.Mutex - active map[*cooldownRecorder]struct{} -} - -// cooldownRecorder collects when the versions withheld during one rewrite -// become available. -type cooldownRecorder struct { - // earliest is when the first withheld version becomes available, or - // zero if none was withheld. - earliest time.Time - // unknown is set when a version was withheld without a time it becomes - // available. - unknown bool -} - -// watchCooldown starts recording the versions cooldown holds back for a -// rewrite about to run. unwatchCooldown ends it. -func (p *Proxy) watchCooldown() *cooldownRecorder { - rec := &cooldownRecorder{} - w := &p.cooldownWatch - w.mu.Lock() - defer w.mu.Unlock() - if w.active == nil { - w.active = make(map[*cooldownRecorder]struct{}) - } - w.active[rec] = struct{}{} - return rec -} - -func (p *Proxy) unwatchCooldown(rec *cooldownRecorder) { - w := &p.cooldownWatch - w.mu.Lock() - defer w.mu.Unlock() - delete(w.active, rec) -} - -// cooldownExpiry returns when a rewrite that started at start and recorded -// into rec should expire: when the first version it held back leaves -// cooldown, and no later than cooldownRewriteTTL after start. -func (p *Proxy) cooldownExpiry(rec *cooldownRecorder, start time.Time) time.Time { - w := &p.cooldownWatch - w.mu.Lock() - earliest, unknown := rec.earliest, rec.unknown - w.mu.Unlock() - - limit := start.Add(p.cooldownRewriteTTL()) - if unknown || earliest.IsZero() || earliest.After(limit) { - return limit - } - return earliest -} - -// withheld tells the running rewrites that a version is held back until -// availableAt, zero when that is unknown. -func (p *Proxy) withheld(availableAt time.Time) { - w := &p.cooldownWatch - w.mu.Lock() - defer w.mu.Unlock() - for rec := range w.active { - switch { - case availableAt.IsZero(): - rec.unknown = true - case rec.earliest.IsZero() || availableAt.Before(rec.earliest): - rec.earliest = availableAt - } - } -} - -// cooldown returns the proxy's cooldown policy, telling the running rewrites -// about each version it holds back. Handlers use it rather than p.Cooldown. -func (p *Proxy) cooldown() CooldownPolicy { //nolint:ireturn // nil when cooldown is not configured, as p.Cooldown is - if p.Cooldown == nil { - return nil - } - return recordingCooldown{p: p} -} - -// recordingCooldown is the proxy's cooldown policy, reporting the versions it -// holds back to the running rewrites. -type recordingCooldown struct { - p *Proxy -} - -func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { - packagePURL = packageurl.PolicyPackageKey(packagePURL) - if r.p.Cooldown.IsAllowed(ecosystem, packagePURL, publishedAt) { - return true - } - r.p.withheld(r.availableAt(ecosystem, packagePURL, publishedAt)) - return false -} - -func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision { - packagePURL = packageurl.PolicyPackageKey(packagePURL) - d := r.p.Cooldown.Evaluate(ecosystem, packagePURL, publishedAt, evaluatedAt) - if !d.Allowed { - r.p.withheld(d.AvailableAt) - } - return d -} - -func (r recordingCooldown) For(ecosystem, packagePURL string) time.Duration { - return r.p.Cooldown.For(ecosystem, packageurl.PolicyPackageKey(packagePURL)) -} - -func (r recordingCooldown) Enabled() bool { - return r.p.Cooldown.Enabled() -} - -// availableAt is when a version IsAllowed held back leaves cooldown, or zero -// when that is unknown. -func (r recordingCooldown) availableAt(ecosystem, packagePURL string, publishedAt time.Time) time.Time { - if publishedAt.IsZero() { - return time.Time{} - } - d := r.For(ecosystem, packagePURL) - if d <= 0 { - return time.Time{} - } - return publishedAt.Add(d) -} diff --git a/internal/handler/cooldown_expiry_test.go b/internal/handler/cooldown_expiry_test.go deleted file mode 100644 index fd55845..0000000 --- a/internal/handler/cooldown_expiry_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package handler - -import ( - "context" - "net/http" - "os" - "path/filepath" - "regexp" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// TestCachedRewrite_ExpiresWhenWithheldVersionIsAvailable checks that a -// rewrite which held a version back is cached until that version leaves -// cooldown, and no longer. -func TestCachedRewrite_ExpiresWhenWithheldVersionIsAvailable(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - clock := &testClock{t: time.Now()} - proxy.now = clock.now - proxy.SetMetadataRewriteCacheSize(1 << 20) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - purl := canonicalPackagePURL("npm", "left-pad") - // One version is 3d-5h old: it leaves cooldown in five hours. - published := clock.now().Add(-72*time.Hour + 5*time.Hour) - - var calls int - var rewrite keepingRewrite = func(b []byte) ([]byte, []string, error) { - calls++ - if proxy.cooldown().IsAllowed("npm", purl, published) { - return b, []string{"1.0.0", "1.1.0"}, nil - } - return b, []string{"1.0.0"}, nil - } - get := func() { - if _, err := proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", rewrite); err != nil { - t.Fatal(err) - } - } - - get() - clock.advance(5*time.Hour - time.Minute) - get() - if calls != 1 { - t.Errorf("rewrites before the withheld version is available = %d, want 1", calls) - } - clock.advance(time.Minute) - get() - if calls != 2 { - t.Errorf("rewrites once the withheld version is available = %d, want 2", calls) - } -} - -// TestCachedRewrite_NothingWithheldCachedForCap checks that a rewrite which -// held nothing back is cached for cooldownRewriteTTL. -func TestCachedRewrite_NothingWithheldCachedForCap(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} - proxy.now = clock.now - proxy.SetMetadataRewriteCacheSize(1 << 20) - proxy.Cooldown = &cooldown.Config{Default: "2h"} - proxy.ShortestCooldown = 2 * time.Hour - var r countingRewrite - - get := func() { - _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) - } - get() - clock.advance(2*time.Hour - time.Second) - get() - if got := r.calls.Load(); got != 1 { - t.Errorf("rewrites within the cap = %d, want 1", got) - } - clock.advance(time.Second) - get() - if got := r.calls.Load(); got != 2 { - t.Errorf("rewrites after the cap = %d, want 2", got) - } -} - -// TestCooldownExpiry checks how one rewrite's recorded decisions set its -// expiry, including decisions made for other rewrites running at the same -// time, which may only make it expire earlier. -func TestCooldownExpiry(t *testing.T) { - p := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}, ShortestCooldown: 72 * time.Hour} - // IsAllowed compares with the wall clock, so the times are relative to it. - start := time.Now() - limit := start.Add(24 * time.Hour) - purl := canonicalPackagePURL("npm", "left-pad") - - rec := p.watchCooldown() - if got := p.cooldownExpiry(rec, start); !got.Equal(limit) { - t.Errorf("nothing withheld: expiry = %v, want the cap %v", got, limit) - } - - other := p.watchCooldown() - p.cooldown().IsAllowed("npm", purl, start.Add(-60*time.Hour)) // available in 12h - p.unwatchCooldown(other) - p.cooldown().IsAllowed("npm", purl, start.Add(-66*time.Hour)) // available in 6h, after other ended - if got, want := p.cooldownExpiry(rec, start), start.Add(6*time.Hour); !got.Equal(want) { - t.Errorf("rec expiry = %v, want the earliest availability %v", got, want) - } - if got, want := p.cooldownExpiry(other, start), start.Add(12*time.Hour); !got.Equal(want) { - t.Errorf("other expiry = %v, want %v: decisions after it ended don't count", got, want) - } - - p.cooldown().Evaluate("npm", purl, start.Add(-71*time.Hour), start) // available in 1h - if got, want := p.cooldownExpiry(rec, start), start.Add(time.Hour); !got.Equal(want) { - t.Errorf("after Evaluate: expiry = %v, want %v", got, want) - } - p.unwatchCooldown(rec) - if len(p.cooldownWatch.active) != 0 { - t.Errorf("%d recorders left after unwatch", len(p.cooldownWatch.active)) - } -} - -// TestHandlersAskCooldownThroughRecorder checks that handlers make cooldown -// decisions through Proxy.cooldown. A decision made on Proxy.Cooldown -// directly would not reach the rewrites running, so a cached rewrite could -// keep hiding a version past its cooldown. -func TestHandlersAskCooldownThroughRecorder(t *testing.T) { - direct := regexp.MustCompile(`\.Cooldown\.(IsAllowed|Evaluate)\(|Cooldown:\s+\w+(\.\w+)*\.Cooldown\b`) - files, err := filepath.Glob("*.go") - if err != nil { - t.Fatal(err) - } - for _, f := range files { - if strings.HasSuffix(f, "_test.go") || f == "cooldown_expiry.go" { - continue - } - src, err := os.ReadFile(f) - if err != nil { - t.Fatal(err) - } - for i, line := range strings.Split(string(src), "\n") { - if direct.MatchString(line) { - t.Errorf("%s:%d asks Proxy.Cooldown directly; use Proxy.cooldown(): %s", f, i+1, strings.TrimSpace(line)) - } - } - } -} - -// TestHandlerRewriteExpiresWithWithheldVersion checks, through the handlers, -// that a cached rewrite expires when the version cooldown held back from it -// becomes available. -func TestHandlerRewriteExpiresWithWithheldVersion(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := cooldownUpstream(t) // the newest version was published 2h ago - proxy, _, clock := validatorProxyWithDB(t, u, time.Hour) - // Cooldown compares with the wall clock, so the proxy's clock - // follows it here. - clock.t = time.Now() - proxy.Cooldown = &cooldown.Config{Default: "3h"} - proxy.ShortestCooldown = 3 * time.Hour - handler := c.handler(proxy, u) - - if rec := serve(handler, c.path, nil); rec.Code != http.StatusOK { - t.Fatalf("status %d", rec.Code) - } - proxy.rewrites.mu.Lock() - defer proxy.rewrites.mu.Unlock() - if len(proxy.rewrites.entries) != 1 { - t.Fatalf("%d cached rewrites, want 1", len(proxy.rewrites.entries)) - } - for _, el := range proxy.rewrites.entries { - expires := el.Value.(*rewriteEntry).expires - // Published 2h ago with a 3h cooldown: available in an hour, - // to the second the publish time was given in. - if want := time.Now().Add(time.Hour); expires.Before(want.Add(-2*time.Second)) || expires.After(want.Add(time.Second)) { - t.Errorf("rewrite expires %v, want about %v when the withheld version is available", expires, want) - } - } - }) - } -} diff --git a/internal/handler/cooldown_fail_closed.go b/internal/handler/cooldown_fail_closed.go deleted file mode 100644 index 0dd2c36..0000000 --- a/internal/handler/cooldown_fail_closed.go +++ /dev/null @@ -1,47 +0,0 @@ -package handler - -import "net/http" - -// cooldownApplies reports whether cooldown withholds new versions of the -// package, taking package overrides and exemptions into account. -func (p *Proxy) cooldownApplies(ecosystem, name string) bool { - return p.cooldownEnabled() && p.cooldown().For(ecosystem, canonicalPackagePURL(ecosystem, name)) > 0 -} - -// mustFilterMetadata reports whether the package's metadata may have -// versions to withhold, by cooldown or by the denylist, and so must not be -// served as upstream sent it. -func (p *Proxy) mustFilterMetadata(ecosystem, name string) bool { - return p.cooldownApplies(ecosystem, name) || - len(p.Denylist.Versions(canonicalPackagePURL(ecosystem, name))) != 0 -} - -// writeCooldownLookupError answers a metadata request whose publish times -// couldn't be read: 503 with Retry-After when the rate limit gate held the -// request, 502 otherwise. -func (p *Proxy) writeCooldownLookupError(w http.ResponseWriter, ecosystem, name string, err error) { - if writeUpstreamHeld(w, err) { - return - } - p.Logger.Warn("cooldown: could not read publish times", - "ecosystem", ecosystem, "package", name, "error", err) - http.Error(w, "failed to check cooldown", http.StatusBadGateway) -} - -// serveCachedDespiteCooldownError serves a download from the cache when its -// publish time couldn't be read, and reports whether it did. A cached file was -// already let through once, so the proxy keeps serving it while the upstream -// is unreachable; only a cache miss fails. -func (p *Proxy) serveCachedDespiteCooldownError(w http.ResponseWriter, r *http.Request, ecosystem, name, version, filename string, lookupErr error) bool { - if p.versionDenied(ecosystem, name, version) { - return false - } - cached, err := p.GetCachedArtifact(r.Context(), ecosystem, name, version, filename) - if err != nil || cached == nil { - return false - } - p.Logger.Warn("cooldown: could not read publish times, serving cached file", - "ecosystem", ecosystem, "package", name, "version", version, "error", lookupErr) - ServeArtifactRequest(w, r, cached) - return true -} diff --git a/internal/handler/cooldown_fail_closed_test.go b/internal/handler/cooldown_fail_closed_test.go deleted file mode 100644 index dc7cb98..0000000 --- a/internal/handler/cooldown_fail_closed_test.go +++ /dev/null @@ -1,313 +0,0 @@ -package handler - -import ( - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - registryclient "github.com/git-pkgs/registries/client" - "github.com/git-pkgs/registries/fetch" -) - -// holdPaths returns a client that answers requests for the given paths with -// the rate limit gate's HeldError and sends the rest to upstream. -func holdPaths(paths ...string) *http.Client { - held := ®istryclient.HeldError{Host: "upstream.example", Until: time.Now().Add(30 * time.Second), RetryAfter: 30 * time.Second} - return &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) { - for _, p := range paths { - if r.URL.Path == p { - return nil, held - } - } - return http.DefaultTransport.RoundTrip(r) - })} -} - -func serveGet(t *testing.T, h http.Handler, path string) *httptest.ResponseRecorder { - t.Helper() - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - return w -} - -func wantStatus(t *testing.T, w *httptest.ResponseRecorder, status int, leaked string) { - t.Helper() - if w.Code != status { - t.Errorf("status = %d, want %d: %s", w.Code, status, w.Body.String()) - } - if status == http.StatusServiceUnavailable && w.Header().Get("Retry-After") == "" { - t.Error("503 without Retry-After") - } - if leaked != "" && strings.Contains(w.Body.String(), leaked) { - t.Errorf("response lists %q, which cooldown should have withheld", leaked) - } -} - -func TestGemCooldownFailsClosed(t *testing.T) { - index := "---\n1.0.0 |checksum:abc\n2.0.0 |checksum:def\n" - versionsStatus := http.StatusInternalServerError - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/api/v1/versions/") { - w.WriteHeader(versionsStatus) - return - } - _, _ = io.WriteString(w, index) - })) - defer upstream.Close() - - t.Run("versions API fails", func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - h := NewGemHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) - wantStatus(t, serveGet(t, h.Routes(), "/info/testgem"), http.StatusBadGateway, "2.0.0") - }) - - t.Run("versions API held by rate limit", func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - proxy.HTTPClient = holdPaths("/api/v1/versions/testgem.json") - h := NewGemHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) - wantStatus(t, serveGet(t, h.Routes(), "/info/testgem"), http.StatusServiceUnavailable, "2.0.0") - }) - - t.Run("exempt package needs no times", func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "3d", Packages: map[string]string{"pkg:gem/testgem": "0"}} - h := NewGemHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) - w := serveGet(t, h.Routes(), "/info/testgem") - if w.Code != http.StatusOK || w.Body.String() != index { - t.Errorf("status = %d, body = %q; want the index unchanged", w.Code, w.Body.String()) - } - }) -} - -func TestHexCooldownFailsClosed(t *testing.T) { - now := time.Now() - proto := buildHexSigned(buildHexPackage("testpkg", []string{testVersion100, "2.0.0"})) - apiJSON, _ := json.Marshal(hexPackageAPI{Releases: []hexRelease{ - {Version: testVersion100, InsertedAt: now.Add(-7 * 24 * time.Hour).Format(time.RFC3339Nano)}, - {Version: "2.0.0", InsertedAt: now.Add(-time.Hour).Format(time.RFC3339Nano)}, - }}) - - newUpstream := func(t *testing.T, packageBody []byte, apiStatus int) *httptest.Server { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/packages/testpkg": - _, _ = w.Write(packageBody) - case "/api/packages/testpkg": - w.WriteHeader(apiStatus) - _, _ = w.Write(apiJSON) - default: - w.WriteHeader(http.StatusNotFound) - } - })) - t.Cleanup(srv.Close) - return srv - } - handler := func(t *testing.T, srv *httptest.Server, client *http.Client) http.Handler { - proxy, _, _, _ := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - if client != nil { - proxy.HTTPClient = client - } - return NewHexHandlerWithUpstreams(proxy, "http://proxy.test", srv.URL, srv.URL).Routes() - } - - t.Run("API fails", func(t *testing.T) { - srv := newUpstream(t, proto, http.StatusInternalServerError) - wantStatus(t, serveGet(t, handler(t, srv, nil), "/packages/testpkg"), http.StatusBadGateway, "") - }) - - t.Run("API held by rate limit", func(t *testing.T) { - srv := newUpstream(t, proto, http.StatusOK) - w := serveGet(t, handler(t, srv, holdPaths("/api/packages/testpkg")), "/packages/testpkg") - wantStatus(t, w, http.StatusServiceUnavailable, "") - }) - - t.Run("package can't be filtered", func(t *testing.T) { - srv := newUpstream(t, []byte("not gzip 2.0.0"), http.StatusOK) - wantStatus(t, serveGet(t, handler(t, srv, nil), "/packages/testpkg"), http.StatusBadGateway, "2.0.0") - }) - - t.Run("filtered", func(t *testing.T) { - srv := newUpstream(t, proto, http.StatusOK) - w := serveGet(t, handler(t, srv, nil), "/packages/testpkg") - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200", w.Code) - } - }) -} - -func TestPyPICooldownFailsClosed(t *testing.T) { - simple := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[` + - `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz"},` + - `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz"}]}` - - tests := []struct { - name string - json func(*http.Request) (*http.Response, error) - status int - }{ - {"JSON API fails", func(r *http.Request) (*http.Response, error) { - return &http.Response{StatusCode: http.StatusInternalServerError, Body: http.NoBody, Request: r}, nil - }, http.StatusBadGateway}, - {"JSON API held by rate limit", func(*http.Request) (*http.Response, error) { - return nil, ®istryclient.HeldError{Host: "pypi.test", RetryAfter: 30 * time.Second} - }, http.StatusServiceUnavailable}, - {"JSON has no releases", func(r *http.Request) (*http.Response, error) { - return pypiHTTPResponse(r, "application/json", `{"info":{}}`), nil - }, http.StatusBadGateway}, - {"JSON API doesn't know the package", func(r *http.Request) (*http.Response, error) { - return &http.Response{StatusCode: http.StatusNotFound, Body: http.NoBody, Request: r}, nil - }, http.StatusOK}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - if r.URL.Path == "/pypi/ruff/json" { - return tt.json(r) - } - return pypiHTTPResponse(r, pypiSimpleJSON, simple), nil - }) - proxy.Cooldown = &cooldown.Config{Default: "7d"} - - req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) - req.Header.Set("Accept", uvPyPIAccept) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - leaked := "ruff-2.0.0" - if tt.status == http.StatusOK { - leaked = "" - } - wantStatus(t, w, tt.status, leaked) - }) - } -} - -func TestPyPIDownloadCooldownFailsClosed(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer upstream.Close() - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "7d"} - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("sdist")), ContentType: "application/octet-stream"} - h := &PyPIHandler{proxy: proxy, upstreamURL: upstream.URL, proxyURL: "http://localhost"} - - w := serveGet(t, h.Routes(), "/packages/packages/ab/cd/ef0123456789/newpkg-2.0.0.tar.gz") - wantStatus(t, w, http.StatusBadGateway, "") - if fetcher.fetchCalled { - t.Error("fetched a file whose publish time couldn't be checked") - } -} - -func TestNPMDownloadCooldownFailsClosed(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer upstream.Close() - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "7d"} - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("tarball")), ContentType: "application/octet-stream"} - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - - w := serveGet(t, h.Routes(), "/leftpad/-/leftpad-2.0.0.tgz") - wantStatus(t, w, http.StatusBadGateway, "") - if fetcher.fetchCalled { - t.Error("fetched a tarball whose publish time couldn't be checked") - } -} - -// TestDownloadCooldownErrorServesCachedFile checks that a file already in the -// cache is still served when its publish time can't be read, while an uncached -// one fails. -func TestDownloadCooldownErrorServesCachedFile(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer upstream.Close() - - tests := []struct { - ecosystem, name, version, filename, path string - handler func(*Proxy) http.Handler - }{ - {"pypi", "newpkg", "2.0.0", "newpkg-2.0.0.tar.gz", "/packages/packages/ab/cd/ef0123456789/newpkg-2.0.0.tar.gz", - func(p *Proxy) http.Handler { - return (&PyPIHandler{proxy: p, upstreamURL: upstream.URL, downloadURL: upstream.URL, proxyURL: "http://localhost"}).Routes() - }}, - {"npm", "leftpad", "2.0.0", "leftpad-2.0.0.tgz", "/leftpad/-/leftpad-2.0.0.tgz", - func(p *Proxy) http.Handler { return NewNPMHandler(p, "http://proxy.test", upstream.URL).Routes() }}, - } - for _, tt := range tests { - t.Run(tt.ecosystem, func(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "7d"} - h := tt.handler(proxy) - - seedPackage(t, db, store, tt.ecosystem, tt.name, tt.version, tt.filename, "cached file") - w := serveGet(t, h, tt.path) - wantStatus(t, w, http.StatusOK, "") - if w.Body.String() != "cached file" { - t.Errorf("body = %q, want the cached file", w.Body.String()) - } - - other := strings.ReplaceAll(tt.path, "2.0.0", "3.0.0") - wantStatus(t, serveGet(t, h, other), http.StatusBadGateway, "") - if fetcher.fetchCalled { - t.Error("fetched a file whose publish time couldn't be checked") - } - }) - } -} - -// TestCooldownRewriteFailureFailsClosed checks that a metadata document the -// cooldown rewrite can't parse is refused rather than passed on, and still -// passed on when cooldown doesn't apply. -func TestCooldownRewriteFailureFailsClosed(t *testing.T) { - const broken = `{"name": "pkg", "versions": [ 2.0.0` - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set(headerContentType, contentTypeJSON) - _, _ = io.WriteString(w, broken) - })) - defer upstream.Close() - - tests := []struct { - name string - purl string - handler func(*Proxy) http.Handler - path string - }{ - {"npm", "pkg:npm/pkg", func(p *Proxy) http.Handler { return NewNPMHandler(p, "http://proxy.test", upstream.URL).Routes() }, "/pkg"}, - {"composer", "pkg:composer/vendor/pkg", func(p *Proxy) http.Handler { - return NewComposerHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL).Routes() - }, "/p2/vendor/pkg.json"}, - {"pub", "pkg:pub/pkg", func(p *Proxy) http.Handler { - return NewPubHandlerWithUpstream(p, "http://proxy.test", upstream.URL).Routes() - }, "/api/packages/pkg"}, - {"pypi json", "pkg:pypi/pkg", func(p *Proxy) http.Handler { - return NewPyPIHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, "").Routes() - }, "/pypi/pkg/json"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "7d"} - wantStatus(t, serveGet(t, tt.handler(proxy), tt.path), http.StatusBadGateway, "2.0.0") - }) - t.Run(tt.name+" exempt", func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "7d", Packages: map[string]string{tt.purl: "0"}} - w := serveGet(t, tt.handler(proxy), tt.path) - if w.Code != http.StatusOK { - t.Errorf("status = %d, want 200 for a package cooldown doesn't apply to", w.Code) - } - }) - } -} diff --git a/internal/handler/cooldown_metadata_cache_test.go b/internal/handler/cooldown_metadata_cache_test.go deleted file mode 100644 index 1eb4a86..0000000 --- a/internal/handler/cooldown_metadata_cache_test.go +++ /dev/null @@ -1,644 +0,0 @@ -package handler - -import ( - "bytes" - "compress/gzip" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// cooldownCountingUpstream serves fixed documents by path with ETags, answers a -// matching If-None-Match with 304, and counts what it was asked. Setting -// down makes it answer 500, and closing gate holds requests until opened. -type cooldownCountingUpstream struct { - *httptest.Server - mu sync.Mutex - docs map[string][]byte - gzipped map[string]bool - requests map[string]int - notModified map[string]int - down map[string]bool - gate chan struct{} -} - -func newCooldownCountingUpstream(t *testing.T) *cooldownCountingUpstream { - t.Helper() - u := &cooldownCountingUpstream{ - docs: map[string][]byte{}, gzipped: map[string]bool{}, - requests: map[string]int{}, notModified: map[string]int{}, down: map[string]bool{}, - } - u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - u.mu.Lock() - u.requests[r.URL.Path]++ - gate := u.gate - doc, ok := u.docs[r.URL.Path] - down := u.down[r.URL.Path] - gz := u.gzipped[r.URL.Path] - u.mu.Unlock() - if gate != nil { - <-gate - } - switch { - case down: - w.WriteHeader(http.StatusInternalServerError) - return - case !ok: - w.WriteHeader(http.StatusNotFound) - return - } - sum := sha256.Sum256(doc) - etag := `"` + hex.EncodeToString(sum[:8]) + `"` - w.Header().Set("ETag", etag) - if r.Header.Get("If-None-Match") == etag { - u.mu.Lock() - u.notModified[r.URL.Path]++ - u.mu.Unlock() - w.WriteHeader(http.StatusNotModified) - return - } - if gz && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { - w.Header().Set("Content-Encoding", "gzip") - zw := gzip.NewWriter(w) - _, _ = zw.Write(doc) - _ = zw.Close() - return - } - _, _ = w.Write(doc) - })) - t.Cleanup(u.Close) - return u -} - -func (u *cooldownCountingUpstream) set(path string, doc []byte) { - u.mu.Lock() - defer u.mu.Unlock() - u.docs[path] = doc -} - -func (u *cooldownCountingUpstream) setDown(path string, down bool) { - u.mu.Lock() - defer u.mu.Unlock() - u.down[path] = down -} - -func (u *cooldownCountingUpstream) counts(path string) (requests, notModified int) { - u.mu.Lock() - defer u.mu.Unlock() - return u.requests[path], u.notModified[path] -} - -// cooldownCacheProxy returns a proxy with cooldown and the metadata cache on. -func cooldownCacheProxy(t *testing.T, u *cooldownCountingUpstream) *Proxy { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = u.Client() - config := &cooldown.Config{Default: "3d"} - // Config parses itself on first use, which isn't safe to race. - config.Enabled() - proxy.Cooldown = config - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - return proxy -} - -// expireCachedMetadata moves a metadata cache row's fetch time past the TTL. -func expireCachedMetadata(t *testing.T, proxy *Proxy, ecosystem, key string) { - t.Helper() - entry, err := proxy.DB.GetMetadataCache(ecosystem, key) - if err != nil || entry == nil { - t.Fatalf("metadata entry %s/%s: %v", ecosystem, key, err) - } - entry.FetchedAt.Time = time.Now().Add(-2 * proxy.MetadataTTL) - if err := proxy.DB.UpsertMetadataCache(entry); err != nil { - t.Fatalf("expiring metadata entry: %v", err) - } -} - -func getWith(t *testing.T, h http.Handler, path string, header http.Header) *httptest.ResponseRecorder { - t.Helper() - req := httptest.NewRequest(http.MethodGet, path, nil) - for k, vv := range header { - req.Header[k] = vv - } - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - return w -} - -const ( - gemInfoPath = "/info/testgem" - gemVersionsPath = "/api/v1/versions/testgem.json" -) - -func gemVersionsJSON(t *testing.T, versions map[string]time.Duration) []byte { - t.Helper() - var list []gemVersion - for number, age := range versions { - list = append(list, gemVersion{Number: number, Platform: "ruby", CreatedAt: time.Now().Add(-age).Format(time.RFC3339)}) - } - b, err := json.Marshal(list) - if err != nil { - t.Fatal(err) - } - return b -} - -// gemCacheSetup serves testgem with 1.0.0 published a week ago and 2.0.0 an -// hour ago. -func gemCacheSetup(t *testing.T) (*cooldownCountingUpstream, *Proxy, http.Handler) { - t.Helper() - u := newCooldownCountingUpstream(t) - u.set(gemInfoPath, []byte("---\n1.0.0 |checksum:a\n2.0.0 |checksum:b\n")) - u.set(gemVersionsPath, gemVersionsJSON(t, map[string]time.Duration{"1.0.0": 7 * 24 * time.Hour, "2.0.0": time.Hour})) - proxy := cooldownCacheProxy(t, u) - return u, proxy, NewGemHandlerWithUpstream(proxy, "http://proxy.test", u.URL).Routes() -} - -const gemFiltered = "---\n1.0.0 |checksum:a\n" - -func TestGemCooldownServesFromMetadataCache(t *testing.T) { - u, _, h := gemCacheSetup(t) - for i := range 3 { - w := getWith(t, h, gemInfoPath, nil) - if w.Code != http.StatusOK || w.Body.String() != gemFiltered { - t.Fatalf("request %d: status %d, body %q", i, w.Code, w.Body) - } - } - for _, path := range []string{gemInfoPath, gemVersionsPath} { - if n, _ := u.counts(path); n != 1 { - t.Errorf("%s requested %d times, want 1 within the TTL", path, n) - } - } -} - -func TestGemCooldownRevalidatesAfterTTL(t *testing.T) { - u, proxy, h := gemCacheSetup(t) - getWith(t, h, gemInfoPath, nil) - expireCachedMetadata(t, proxy, "gem", "info/testgem") - expireCachedMetadata(t, proxy, "gem", "api/v1/versions/testgem.json") - - w := getWith(t, h, gemInfoPath, nil) - if w.Code != http.StatusOK || w.Body.String() != gemFiltered { - t.Fatalf("status %d, body %q", w.Code, w.Body) - } - for _, path := range []string{gemInfoPath, gemVersionsPath} { - if n, nm := u.counts(path); n != 2 || nm != 1 { - t.Errorf("%s: %d requests, %d answered 304; want 2 and 1", path, n, nm) - } - } -} - -func TestGemCooldownConcurrentRequestsShareFetch(t *testing.T) { - u, proxy, h := gemCacheSetup(t) - u.gate = make(chan struct{}) - var release sync.Once - t.Cleanup(func() { release.Do(func() { close(u.gate) }) }) - - const clients = 5 - var wg sync.WaitGroup - codes := make([]int, clients) - bodies := make([]string, clients) - for i := range clients { - wg.Go(func() { - w := getWith(t, h, gemInfoPath, nil) - codes[i], bodies[i] = w.Code, w.Body.String() - }) - } - waitForMetadataWaiters(t, proxy, metadataCoalesceKey("gem", "info/testgem", u.URL+gemInfoPath, "*/*", "identity", false), clients-1) - waitForMetadataWaiters(t, proxy, metadataCoalesceKey("gem", "api/v1/versions/testgem.json", u.URL+gemVersionsPath, contentTypeJSON, "", true), clients-1) - release.Do(func() { close(u.gate) }) - wg.Wait() - - for i := range clients { - if codes[i] != http.StatusOK || bodies[i] != gemFiltered { - t.Errorf("client %d: status %d, body %q", i, codes[i], bodies[i]) - } - } - for _, path := range []string{gemInfoPath, gemVersionsPath} { - if n, _ := u.counts(path); n != 1 { - t.Errorf("%s requested %d times, want 1", path, n) - } - } -} - -func TestGemCooldownStaleCopyIsFiltered(t *testing.T) { - u, proxy, h := gemCacheSetup(t) - getWith(t, h, gemInfoPath, nil) - expireCachedMetadata(t, proxy, "gem", "info/testgem") - expireCachedMetadata(t, proxy, "gem", "api/v1/versions/testgem.json") - u.setDown(gemInfoPath, true) - u.setDown(gemVersionsPath, true) - - w := getWith(t, h, gemInfoPath, nil) - if w.Code != http.StatusOK || w.Body.String() != gemFiltered { - t.Fatalf("status %d, body %q; want the stale copy, filtered", w.Code, w.Body) - } - if got := w.Header().Get("Warning"); got != staleWarning { - t.Errorf("Warning = %q, want %q", got, staleWarning) - } -} - -func TestGemCooldownFailsClosedWithoutTimes(t *testing.T) { - u, _, h := gemCacheSetup(t) - u.setDown(gemVersionsPath, true) - - w := getWith(t, h, gemInfoPath, nil) - if w.Code != http.StatusBadGateway || strings.Contains(w.Body.String(), "1.0.0") { - t.Fatalf("status %d, body %q; want 502 without the index", w.Code, w.Body) - } - // The index alone was cached; it must not be served unfiltered either. - w = getWith(t, h, gemInfoPath, nil) - if w.Code != http.StatusBadGateway { - t.Fatalf("second request: status %d, want 502", w.Code) - } -} - -func TestGemCooldownClientRevalidates(t *testing.T) { - _, _, h := gemCacheSetup(t) - first := getWith(t, h, gemInfoPath, nil) - etag := first.Header().Get("ETag") - if etag == "" { - t.Fatal("filtered index served without an ETag") - } - w := getWith(t, h, gemInfoPath, http.Header{"If-None-Match": {etag}}) - if w.Code != http.StatusNotModified { - t.Fatalf("status %d, want 304", w.Code) - } -} - -// The index can be fetched again while the versions API response is still -// within its TTL. A version published in between is then in the index but -// has no publish time, and must not be let through for that. -func TestGemCooldownRefreshesTimesOlderThanIndex(t *testing.T) { - for _, timesDown := range []bool{false, true} { - name := "times refreshed" - if timesDown { - name = "times upstream down" - } - t.Run(name, func(t *testing.T) { - u, proxy, h := gemCacheSetup(t) - getWith(t, h, gemInfoPath, nil) - - u.set(gemInfoPath, []byte("---\n1.0.0 |checksum:a\n2.0.0 |checksum:b\n3.0.0 |checksum:c\n")) - u.set(gemVersionsPath, gemVersionsJSON(t, map[string]time.Duration{ - "1.0.0": 7 * 24 * time.Hour, "2.0.0": time.Hour, "3.0.0": time.Minute, - })) - u.setDown(gemVersionsPath, timesDown) - expireCachedMetadata(t, proxy, "gem", "info/testgem") - - w := getWith(t, h, gemInfoPath, nil) - if w.Code != http.StatusOK || w.Body.String() != gemFiltered { - t.Fatalf("status %d, body %q; want %q", w.Code, w.Body, gemFiltered) - } - if n, _ := u.counts(gemVersionsPath); n != 2 { - t.Errorf("versions API requested %d times, want 2", n) - } - if got := w.Header().Get("Warning"); (got != "") != timesDown { - t.Errorf("Warning = %q with times upstream down = %v", got, timesDown) - } - }) - } -} - -const ( - hexPackagePath = "/packages/testpkg" - hexAPIPath = "/api/packages/testpkg" -) - -func hexAPIJSON(t *testing.T, releases map[string]time.Duration) []byte { - t.Helper() - var api hexPackageAPI - for version, age := range releases { - api.Releases = append(api.Releases, hexRelease{Version: version, InsertedAt: time.Now().Add(-age).Format(time.RFC3339Nano)}) - } - b, err := json.Marshal(api) - if err != nil { - t.Fatal(err) - } - return b -} - -func hexCacheSetup(t *testing.T, recent time.Duration) (*cooldownCountingUpstream, *Proxy, http.Handler, []byte) { - t.Helper() - u := newCooldownCountingUpstream(t) - signed := buildHexSigned(buildHexPackage("testpkg", []string{testVersion100, "2.0.0"})) - u.set(hexPackagePath, signed) - u.set(hexAPIPath, hexAPIJSON(t, map[string]time.Duration{testVersion100: 7 * 24 * time.Hour, "2.0.0": recent})) - proxy := cooldownCacheProxy(t, u) - return u, proxy, NewHexHandlerWithUpstreams(proxy, "http://proxy.test", u.URL, u.URL).Routes(), signed -} - -// hexVersions decodes the release versions of a served package. -func hexVersions(t *testing.T, body []byte) []string { - t.Helper() - zr, err := gzip.NewReader(bytes.NewReader(body)) - if err != nil { - t.Fatal(err) - } - signed, err := io.ReadAll(zr) - if err != nil { - t.Fatal(err) - } - payload, err := extractProtobufBytes(signed, 1) - if err != nil { - t.Fatal(err) - } - versions, err := packageReleaseVersions(payload) - if err != nil { - t.Fatal(err) - } - return versions -} - -func TestHexCooldownServesFromMetadataCache(t *testing.T) { - u, proxy, h, _ := hexCacheSetup(t, time.Hour) - for i := range 3 { - w := getWith(t, h, hexPackagePath, nil) - if w.Code != http.StatusOK { - t.Fatalf("request %d: status %d", i, w.Code) - } - if got := hexVersions(t, w.Body.Bytes()); len(got) != 1 || got[0] != testVersion100 { - t.Fatalf("request %d: versions %v, want [%s]", i, got, testVersion100) - } - if ce := w.Header().Get("Content-Encoding"); ce != "gzip" { - t.Errorf("Content-Encoding = %q, want gzip", ce) - } - } - for _, path := range []string{hexPackagePath, hexAPIPath} { - if n, _ := u.counts(path); n != 1 { - t.Errorf("%s requested %d times, want 1 within the TTL", path, n) - } - } - - expireCachedMetadata(t, proxy, "hex", "packages/testpkg") - expireCachedMetadata(t, proxy, "hex", hashedCacheKey(u.URL, "api/packages/testpkg")) - if w := getWith(t, h, hexPackagePath, nil); w.Code != http.StatusOK || len(hexVersions(t, w.Body.Bytes())) != 1 { - t.Fatalf("after TTL: status %d", w.Code) - } - for _, path := range []string{hexPackagePath, hexAPIPath} { - if n, nm := u.counts(path); n != 2 || nm != 1 { - t.Errorf("%s: %d requests, %d answered 304; want 2 and 1", path, n, nm) - } - } -} - -func TestHexCooldownStaleCopyIsFiltered(t *testing.T) { - u, proxy, h, _ := hexCacheSetup(t, time.Hour) - getWith(t, h, hexPackagePath, nil) - expireCachedMetadata(t, proxy, "hex", "packages/testpkg") - expireCachedMetadata(t, proxy, "hex", hashedCacheKey(u.URL, "api/packages/testpkg")) - u.setDown(hexPackagePath, true) - u.setDown(hexAPIPath, true) - - w := getWith(t, h, hexPackagePath, nil) - if w.Code != http.StatusOK { - t.Fatalf("status %d, want the stale copy", w.Code) - } - if got := hexVersions(t, w.Body.Bytes()); len(got) != 1 || got[0] != testVersion100 { - t.Errorf("versions %v, want [%s]", got, testVersion100) - } - if got := w.Header().Get("Warning"); got != staleWarning { - t.Errorf("Warning = %q, want %q", got, staleWarning) - } -} - -func TestHexCooldownFailsClosedWithoutTimes(t *testing.T) { - u, _, h, _ := hexCacheSetup(t, time.Hour) - u.setDown(hexAPIPath, true) - for i := range 2 { - if w := getWith(t, h, hexPackagePath, nil); w.Code != http.StatusBadGateway { - t.Fatalf("request %d: status %d, want 502", i, w.Code) - } - } -} - -func TestHexCooldownUnfilteredKeepsSignature(t *testing.T) { - _, _, h, signed := hexCacheSetup(t, 7*24*time.Hour) - w := getWith(t, h, hexPackagePath, nil) - if w.Code != http.StatusOK || !bytes.Equal(w.Body.Bytes(), signed) { - t.Fatalf("status %d; want upstream's signed bytes when nothing is in cooldown", w.Code) - } - etag := w.Header().Get("ETag") - if etag == "" { - t.Fatal("package served without an ETag") - } - if w := getWith(t, h, hexPackagePath, http.Header{"If-None-Match": {etag}}); w.Code != http.StatusNotModified { - t.Fatalf("revalidation: status %d, want 304", w.Code) - } -} - -const condaRepodataPath = "/conda-forge/noarch/repodata.json" - -func condaCacheSetup(t *testing.T) (*cooldownCountingUpstream, *Proxy, *CondaHandler, http.Handler) { - t.Helper() - u := newCooldownCountingUpstream(t) - repodata, err := json.Marshal(map[string]any{ - "info": map[string]any{"subdir": "noarch"}, - "packages": map[string]any{ - "old-1.0.0-0.tar.bz2": map[string]any{"name": "old", "version": "1.0.0", "timestamp": time.Now().Add(-7 * 24 * time.Hour).UnixMilli()}, - "new-2.0.0-0.tar.bz2": map[string]any{"name": "new", "version": "2.0.0", "timestamp": time.Now().Add(-time.Hour).UnixMilli()}, - }, - "packages.conda": map[string]any{}, - }) - if err != nil { - t.Fatal(err) - } - u.set(condaRepodataPath, repodata) - u.gzipped[condaRepodataPath] = true - proxy := cooldownCacheProxy(t, u) - h := NewCondaHandlerWithUpstream(proxy, "http://proxy.test", u.URL) - return u, proxy, h, h.Routes() -} - -func condaPackages(t *testing.T, w *httptest.ResponseRecorder) []string { - t.Helper() - if w.Code != http.StatusOK { - t.Fatalf("status %d, body %q", w.Code, w.Body) - } - var doc struct { - Packages map[string]json.RawMessage `json:"packages"` - } - if err := json.Unmarshal(w.Body.Bytes(), &doc); err != nil { - t.Fatalf("decoding repodata: %v", err) - } - var names []string - for name := range doc.Packages { - names = append(names, name) - } - return names -} - -func assertCondaFiltered(t *testing.T, w *httptest.ResponseRecorder) { - t.Helper() - if got := condaPackages(t, w); len(got) != 1 || got[0] != "old-1.0.0-0.tar.bz2" { - t.Fatalf("packages %v, want only old-1.0.0-0.tar.bz2", got) - } -} - -func TestCondaCooldownServesFromMetadataCache(t *testing.T) { - u, proxy, h, routes := condaCacheSetup(t) - for range 3 { - assertCondaFiltered(t, getWith(t, routes, condaRepodataPath, nil)) - } - if n, _ := u.counts(condaRepodataPath); n != 1 { - t.Errorf("repodata requested %d times, want 1 within the TTL", n) - } - - expireCachedMetadata(t, proxy, "conda", h.metadataCacheKey(condaRepodataPath)) - assertCondaFiltered(t, getWith(t, routes, condaRepodataPath, nil)) - if n, nm := u.counts(condaRepodataPath); n != 2 || nm != 1 { - t.Errorf("%d requests, %d answered 304; want 2 and 1", n, nm) - } -} - -func TestCondaCooldownStaleCopyIsFiltered(t *testing.T) { - u, proxy, h, routes := condaCacheSetup(t) - assertCondaFiltered(t, getWith(t, routes, condaRepodataPath, nil)) - expireCachedMetadata(t, proxy, "conda", h.metadataCacheKey(condaRepodataPath)) - u.setDown(condaRepodataPath, true) - - w := getWith(t, routes, condaRepodataPath, nil) - assertCondaFiltered(t, w) - if got := w.Header().Get("Warning"); got != staleWarning { - t.Errorf("Warning = %q, want %q", got, staleWarning) - } -} - -func TestCondaCooldownConcurrentRequestsShareFetch(t *testing.T) { - u, proxy, h, routes := condaCacheSetup(t) - u.gate = make(chan struct{}) - var release sync.Once - t.Cleanup(func() { release.Do(func() { close(u.gate) }) }) - - const clients = 5 - var wg sync.WaitGroup - recorders := make([]*httptest.ResponseRecorder, clients) - for i := range clients { - wg.Go(func() { recorders[i] = getWith(t, routes, condaRepodataPath, nil) }) - } - key := metadataCoalesceKey("conda", h.metadataCacheKey(condaRepodataPath), u.URL+condaRepodataPath, "*/*", "gzip", false) - waitForMetadataWaiters(t, proxy, key, clients-1) - release.Do(func() { close(u.gate) }) - wg.Wait() - - for _, w := range recorders { - assertCondaFiltered(t, w) - } - if n, _ := u.counts(condaRepodataPath); n != 1 { - t.Errorf("repodata requested %d times, want 1", n) - } -} - -func TestCondaCooldownClientRevalidates(t *testing.T) { - _, _, h, routes := condaCacheSetup(t) - // The first copy is streamed as it is filtered, before its ETag is known. - assertCondaFiltered(t, getWith(t, routes, condaRepodataPath, nil)) - w := getWith(t, routes, condaRepodataPath, nil) - assertCondaFiltered(t, w) - etag := w.Header().Get("ETag") - if etag == "" { - t.Fatal("filtered repodata served without an ETag") - } - if w := getWith(t, routes, condaRepodataPath, http.Header{"If-None-Match": {etag}}); w.Code != http.StatusNotModified { - t.Fatalf("revalidation: status %d, want 304", w.Code) - } - - // Once the entry left out leaves cooldown, the copy changes and so does - // its ETag. - key := h.metadataCacheKey(condaRepodataPath) - h.repodataMu.Lock() - state := h.repodataStates[key] - h.repodataMu.Unlock() - if want := time.Now().Add(-time.Hour + 3*24*time.Hour); state.until.Sub(want).Abs() > time.Minute { - t.Errorf("ETag kept until %v, want about %v", state.until, want) - } - req := httptest.NewRequest(http.MethodGet, condaRepodataPath, nil) - req.Header.Set("If-None-Match", etag) - if h.repodataNotModified(httptest.NewRecorder(), req, key, state.digest, state.until) { - t.Error("ETag still matched after the withheld entry left cooldown") - } -} - -func TestCondaCooldownWithoutMetadataCacheStreams(t *testing.T) { - u, proxy, _, routes := condaCacheSetup(t) - proxy.CacheMetadata = false - for range 2 { - assertCondaFiltered(t, getWith(t, routes, condaRepodataPath, nil)) - } - if n, _ := u.counts(condaRepodataPath); n != 2 { - t.Errorf("repodata requested %d times, want 2 without cache_metadata", n) - } -} - -// A cached repodata.json is filtered as it is read from storage: the -// response starts while the stored document is still being read. -func TestCondaCooldownStreamsCachedRepodata(t *testing.T) { - u, proxy, _, routes := condaCacheSetup(t) - packages := map[string]any{} - for i := range 20000 { - packages[fmt.Sprintf("pkg%d-1.0.0-0.tar.bz2", i)] = map[string]any{ - "name": fmt.Sprintf("pkg%d", i), "version": "1.0.0", "timestamp": time.Now().Add(-7 * 24 * time.Hour).UnixMilli(), - } - } - repodata, err := json.Marshal(map[string]any{"info": map[string]any{}, "packages": packages}) - if err != nil { - t.Fatal(err) - } - u.set(condaRepodataPath, repodata) - // Stored uncompressed, so the document is larger than the part the - // storage hands out before stalling. - u.gzipped[condaRepodataPath] = false - if w := getWith(t, routes, condaRepodataPath, nil); w.Code != http.StatusOK { - t.Fatalf("populating cache: status %d", w.Code) - } - - gate := &stallingStorage{Storage: proxy.Storage, head: 256 << 10, release: make(chan struct{})} - proxy.Storage = gate - var release sync.Once - t.Cleanup(func() { release.Do(func() { close(gate.release) }) }) - srv := httptest.NewServer(routes) - t.Cleanup(srv.Close) - - type result struct { - resp *http.Response - err error - } - started := make(chan result, 1) - go func() { - resp, err := http.Get(srv.URL + condaRepodataPath) - if err == nil { - _, err = io.ReadFull(resp.Body, make([]byte, 1024)) - } - started <- result{resp, err} - }() - var res result - select { - case res = <-started: - case <-time.After(5 * time.Second): - release.Do(func() { close(gate.release) }) - if res := <-started; res.resp != nil { - _ = res.resp.Body.Close() - } - t.Fatal("response did not start before the cached document was read to its end") - } - release.Do(func() { close(gate.release) }) - if res.err != nil { - t.Fatal(res.err) - } - _, _ = io.Copy(io.Discard, res.resp.Body) - _ = res.resp.Body.Close() - if n, _ := u.counts(condaRepodataPath); n != 1 { - t.Errorf("repodata requested %d times, want 1", n) - } -} diff --git a/internal/handler/cooldown_patterns_test.go b/internal/handler/cooldown_patterns_test.go deleted file mode 100644 index 71fcc23..0000000 --- a/internal/handler/cooldown_patterns_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package handler - -import ( - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -func TestNuGetCooldownPackagePatterns(t *testing.T) { - for _, tc := range []struct { - name, defaultDuration, pattern string - exact map[string]string - blocked bool - }{ - {"pattern enables cooldown", "", "14d", nil, true}, - {"pattern exempts package", "14d", "0", nil, false}, - {"exact overrides exemption", "", "0", map[string]string{"pkg:nuget/testpkg": "14d"}, true}, - {"exact exempts package", "", "14d", map[string]string{"pkg:nuget/testpkg": "0"}, false}, - } { - t.Run(tc.name, func(t *testing.T) { - p, db, store, fetcher := setupTestProxy(t) - policy, err := cooldown.NewPolicy(&cooldown.Config{Default: tc.defaultDuration, Packages: tc.exact}, map[string]string{"pkg:nuget/test*": tc.pattern}) - if err != nil { - t.Fatal(err) - } - p.Cooldown = policy - seedPackage(t, db, store, "nuget", "testpkg", "2.0.0", "testpkg.2.0.0.nupkg", "cached package") - requests := 0 - upstream := newNuGetCooldownUpstream(t, &requests) - defer upstream.Close() - p.HTTPClient = upstream.Client() - h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) - list := nugetGet(t, h.Routes(), "/v3-flatcontainer/TestPkg/index.json", http.StatusOK) - if strings.Contains(list.Body.String(), "2.0.0") == tc.blocked { - t.Fatalf("incorrect version list: %s", list.Body.String()) - } - status := http.StatusOK - if tc.blocked { - status = http.StatusNotFound - } - nugetGet(t, h.Routes(), "/v3-flatcontainer/TestPkg/2.0.0/testpkg.2.0.0.nupkg", status) - if fetcher.fetchCalled { - t.Fatal("cached or withheld download fetched upstream artifact") - } - }) - } -} - -func TestNPMCooldownPatternExemptionRespectsDenylist(t *testing.T) { - p, db, store, fetcher := setupTestProxy(t) - var err error - p.Cooldown, err = cooldown.NewPolicy(&cooldown.Config{Default: "7d"}, map[string]string{"pkg:npm/@example/*": "0"}) - if err != nil { - t.Fatal(err) - } - setTestDenylist(t, p, "pkg:npm/@example/widget@1.0.0") - published := time.Now().Add(-time.Hour) - for _, version := range []string{"1.0.0", "2.0.0"} { - seedPackage(t, db, store, "npm", "@example/widget", version, "widget-"+version+".tgz", "cached package") - if err := db.SetVersionPublishedAt("pkg:npm/%40example/widget@"+version, "pkg:npm/%40example/widget", published); err != nil { - t.Fatal(err) - } - } - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/@example/widget" { - t.Errorf("unexpected upstream request: %s", r.URL) - } - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"name":"@example/widget","dist-tags":{"latest":"1.0.0"},"versions":{"1.0.0":{},"2.0.0":{}},"time":{"1.0.0":%q,"2.0.0":%q}}`, published.Format(time.RFC3339), published.Format(time.RFC3339)) - })) - defer upstream.Close() - p.HTTPClient = upstream.Client() - h := NewNPMHandler(p, "http://proxy.test", upstream.URL) - server := httptest.NewServer(h.Routes()) - defer server.Close() - for _, tc := range []struct { - path string - status int - contains, absent string - }{ - {"/@example%2Fwidget", http.StatusOK, `"latest":"2.0.0"`, "1.0.0"}, - {"/@example%2Fwidget/-/widget-1.0.0.tgz", http.StatusForbidden, "denylist", "cached package"}, - {"/@example%2Fwidget/-/widget-2.0.0.tgz", http.StatusOK, "cached package", "denylist"}, - } { - response, err := server.Client().Get(server.URL + tc.path) - if err != nil { - t.Fatal(err) - } - body, err := io.ReadAll(response.Body) - _ = response.Body.Close() - if err != nil || response.StatusCode != tc.status || !strings.Contains(string(body), tc.contains) || strings.Contains(string(body), tc.absent) { - t.Errorf("GET %s: status=%d body=%s error=%v", tc.path, response.StatusCode, body, err) - } - } - if fetcher.fetchCalled { - t.Error("cached or denied downloads fetched an upstream artifact") - } -} diff --git a/internal/handler/cooldown_supported_test.go b/internal/handler/cooldown_supported_test.go deleted file mode 100644 index 7187ede..0000000 --- a/internal/handler/cooldown_supported_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package handler - -import ( - "os" - "path/filepath" - "regexp" - "strings" - "testing" -) - -func TestCooldownSupportedCoversHandlers(t *testing.T) { - calls := regexp.MustCompile(`(?:IsAllowed|Evaluate)\("([a-z]+)"|withheld\(h\.proxy, "([a-z]+)"|IsAllowed\((helmMetadataEcosystem)`) - files, err := filepath.Glob("*.go") - if err != nil { - t.Fatal(err) - } - found := map[string]bool{} - for _, f := range files { - if strings.HasSuffix(f, "_test.go") { - continue - } - src, err := os.ReadFile(f) - if err != nil { - t.Fatal(err) - } - for _, m := range calls.FindAllStringSubmatch(string(src), -1) { - if m[3] != "" { - found[helmMetadataEcosystem] = true - continue - } - found[m[1]+m[2]] = true - } - } - if len(found) < 10 { - t.Fatalf("found only %d ecosystems checking cooldown, the pattern no longer matches the handlers", len(found)) - } - for eco := range found { - if !CooldownSupported(eco) { - t.Errorf("%s checks cooldown but is missing from cooldownEcosystems", eco) - } - } - for eco := range cooldownEcosystems { - if !found[eco] { - t.Errorf("%s is in cooldownEcosystems but no handler checks cooldown for it", eco) - } - } -} diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go deleted file mode 100644 index b765f9a..0000000 --- a/internal/handler/cooldown_times.go +++ /dev/null @@ -1,170 +0,0 @@ -package handler - -import ( - "bytes" - "compress/gzip" - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "slices" - "strings" - "time" -) - -// publishTimes maps each version a times document lists to its publish -// time. A version whose time didn't parse maps to the zero time and is let -// through. -type publishTimes map[string]time.Time - -func (p *Proxy) lookupPublishTime(ctx context.Context, t timedMetadata, version string, listed func(metadataResult) ([]string, error)) (time.Time, bool, bool, error) { - var versions []string - docs, err := p.fetchTimedMetadata(ctx, t, func(doc metadataResult) ([]string, error) { - var err error - versions, err = listed(doc) - return versions, err - }) - switch { - case docs.doc.err != nil: - return time.Time{}, false, false, docs.doc.err - case docs.times.err != nil: - return time.Time{}, false, false, docs.times.err - case err != nil: - return time.Time{}, false, false, err - } - published, ok := docs.published[version] - withholdUnknown := !ok && docs.withholdUnknown && slices.Contains(versions, version) - return published, ok && !published.IsZero(), withholdUnknown, nil -} - -// timedMetadata describes metadata whose publish times live in a second -// document, as with the RubyGems compact index and the versions API, or a -// Hex package and the Hex HTTP API. Both are read through the metadata cache. -type timedMetadata struct { - ecosystem string - // docKey and docURL are the cache key and upstream URL of the document - // clients read. They match the plain metadata path's, so both share one - // cached copy. - docKey, docURL string - // timesKey and timesURL are those of the document with publish times. - timesKey, timesURL string - timesAccept string - parseTimes func([]byte) (publishTimes, error) -} - -// timedDocuments is what fetchTimedMetadata hands back. published is read -// from times and is nil when times.err is set. -type timedDocuments struct { - doc metadataResult - times metadataResult - published publishTimes - // withholdUnknown is set when doc lists versions the times document - // doesn't, and that document is a stale copy: those versions are newer - // than the copy, so they are left out rather than let through. - withholdUnknown bool -} - -// fetchTimedMetadata fetches both documents concurrently. The document -// clients read is fetched as the plain metadata path fetches it, with -// identity encoding and Accept */*. -// -// Each document has its own TTL, so a cached times document can be older -// than the document it is used to filter, and miss versions published in -// between. Those versions would be let through as having no publish time. -// listed returns the versions the document lists; when some are missing -// from a times document read from the cache, the times document is fetched -// again before filtering. -func (p *Proxy) fetchTimedMetadata(ctx context.Context, t timedMetadata, listed func(metadataResult) ([]string, error)) (timedDocuments, error) { - var res timedDocuments - var published publishTimes - validate := func(body []byte) error { - parsed, err := t.parseTimes(body) - if err == nil { - published = parsed - } - return err - } - - done := make(chan struct{}) - go func() { - defer close(done) - res.times = p.lookupMetadata(ctx, t.ecosystem, t.timesKey, t.timesURL, t.timesAccept, "", validate) - }() - res.doc = p.lookupMetadata(ctx, t.ecosystem, t.docKey, t.docURL, "*/*", "identity", nil) - <-done - - if res.doc.err != nil || res.times.err != nil { - return res, nil - } - versions, err := listed(res.doc) - if err != nil { - return res, err - } - unknown := func() bool { - for _, v := range versions { - if _, ok := published[v]; !ok { - return true - } - } - return false - } - if unknown() && res.times.fromCache { - p.Logger.Info("cooldown: publish times older than metadata, fetching them again", - "ecosystem", t.ecosystem, "key", t.timesKey) - res.times = p.refreshMetadata(ctx, t.ecosystem, t.timesKey, t.timesURL, t.timesAccept, "", validate) - if res.times.err != nil { - return res, nil - } - } - res.published = published - res.withholdUnknown = res.times.stale && unknown() - return res, nil -} - -// rewriteDigest identifies the inputs of a cooldown rewrite for the rewrite -// cache: both documents and whether unknown versions are withheld. -func (d timedDocuments) rewriteDigest() string { - h := sha256.New() - for _, res := range []metadataResult{d.doc, d.times} { - digest := res.digest - if digest == "" { - sum := sha256.Sum256(res.body) - digest = "sha256:" + hex.EncodeToString(sum[:]) - } - _, _ = fmt.Fprintf(h, "%d:%s", len(digest), digest) - } - _, _ = fmt.Fprintf(h, "%t", d.withholdUnknown) - return "sha256:" + hex.EncodeToString(h.Sum(nil)) -} - -// withheld reports whether cooldown leaves version out, given the -// package's publish times. -func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version string) bool { - published, ok := d.published[version] - if !ok { - return d.withholdUnknown - } - if published.IsZero() { - return false - } - return !p.cooldown().IsAllowed(ecosystem, packagePURL, published) -} - -// decodedMetadataBody returns res's body without its Content-Encoding. The -// documents are fetched with identity encoding, so an upstream that sends -// gzip anyway is the only case to handle. -func (p *Proxy) decodedMetadataBody(res metadataResult) ([]byte, error) { - switch strings.ToLower(strings.TrimSpace(res.contentEncoding)) { - case "", "identity": - return res.body, nil - case "gzip", "x-gzip": - zr, err := gzip.NewReader(bytes.NewReader(res.body)) - if err != nil { - return nil, err - } - defer func() { _ = zr.Close() }() - return p.ReadMetadata(zr) - default: - return nil, fmt.Errorf("unsupported content encoding %q", res.contentEncoding) - } -} diff --git a/internal/handler/cran.go b/internal/handler/cran.go index 10f1750..0ecd2a3 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -3,22 +3,12 @@ package handler import ( "net/http" "strings" - - "github.com/git-pkgs/proxy/internal/retention" ) const ( cranUpstream = "https://cloud.r-project.org" ) -// Source and binary packages are stored under pkg:cran/{name}, the name from -// the filename with its case kept, whether they came from src/contrib, its -// Archive or a binary tree. purl keeps cran names as they are, so package -// rules use the default key mapping. -func init() { - retention.Default.Register(retention.Spec{Key: "cran", CanonicalPackage: retention.DefaultCanonical("cran")}) -} - // CRANHandler handles CRAN (R) registry protocol requests. type CRANHandler struct { proxy *Proxy @@ -35,13 +25,6 @@ func NewCRANHandler(proxy *Proxy, proxyURL string) *CRANHandler { } } -// NewCRANHandlerWithUpstream creates a CRAN handler with a custom upstream. -func NewCRANHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *CRANHandler { - h := NewCRANHandler(proxy, proxyURL) - h.upstreamURL = configuredUpstreamURL(upstreamURL, cranUpstream) - return h -} - // Routes returns the HTTP handler for CRAN requests. func (h *CRANHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -51,39 +34,19 @@ func (h *CRANHandler) Routes() http.Handler { mux.HandleFunc("GET /src/contrib/PACKAGES.gz", h.proxyCached) mux.HandleFunc("GET /src/contrib/PACKAGES.rds", h.proxyCached) - // Repository metadata such as archive.rds and current.rds, used by - // remotes::install_version and pak to find archived versions. - mux.HandleFunc("GET /src/contrib/Meta/{filename}", h.handleMeta) - // Binary package indexes mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES", h.proxyCached) mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES.gz", h.proxyCached) mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES.rds", h.proxyCached) - // macOS binaries for R >= 4.3 use two platform segments, e.g. - // bin/macosx/big-sur-arm64/contrib/4.4/. - mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES", h.proxyCached) - mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES.gz", h.proxyCached) - mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES.rds", h.proxyCached) - // Source package downloads mux.HandleFunc("GET /src/contrib/{filename}", h.handleSourceDownload) mux.HandleFunc("GET /src/contrib/Archive/{name}/{filename}", h.handleSourceDownload) // Binary package downloads mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/{filename}", h.handleBinaryDownload) - mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/{filename}", h.handleBinaryDownload) - return rejectQueryChars(mux) -} - -// handleMeta serves the .rds files under src/contrib/Meta as cached metadata. -func (h *CRANHandler) handleMeta(w http.ResponseWriter, r *http.Request) { - if !strings.HasSuffix(r.PathValue("filename"), ".rds") { - http.NotFound(w, r) - return - } - h.proxyCached(w, r) + return mux } // handleSourceDownload serves a source package, fetching and caching from upstream. @@ -109,19 +72,17 @@ func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Reques result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, version, filename, upstreamURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch package", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) + ServeArtifact(w, result) } // handleBinaryDownload serves a binary package, fetching and caching from upstream. func (h *CRANHandler) handleBinaryDownload(w http.ResponseWriter, r *http.Request) { platform := r.PathValue("platform") - if arch := r.PathValue("arch"); arch != "" { - platform += "-" + arch - } rversion := r.PathValue("rversion") filename := r.PathValue("filename") @@ -146,11 +107,12 @@ func (h *CRANHandler) handleBinaryDownload(w http.ResponseWriter, r *http.Reques result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, storageVersion, filename, upstreamURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch package", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) + ServeArtifact(w, result) } // parseSourceFilename extracts name and version from a CRAN source filename. @@ -188,14 +150,11 @@ func (h *CRANHandler) isBinaryPackage(filename string) bool { return strings.HasSuffix(filename, ".zip") || strings.HasSuffix(filename, ".tgz") } -// metadataCacheKey is the metadata cache key of a request path. -func (h *CRANHandler) metadataCacheKey(urlPath string) string { - return hashedCacheKey(h.upstreamURL, urlPath) -} - // proxyCached forwards a metadata request with caching. func (h *CRANHandler) proxyCached(w http.ResponseWriter, r *http.Request) { - h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "cran", h.metadataCacheKey(r.URL.Path), "*/*") + cacheKey := strings.TrimPrefix(r.URL.Path, "/") + cacheKey = strings.ReplaceAll(cacheKey, "/", "_") + h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "cran", cacheKey, "*/*") } // proxyUpstream forwards a request to CRAN without caching. diff --git a/internal/handler/cran_test.go b/internal/handler/cran_test.go index 6c156c5..6a9c151 100644 --- a/internal/handler/cran_test.go +++ b/internal/handler/cran_test.go @@ -1,14 +1,8 @@ package handler import ( - "io" "log/slog" - "net/http" - "net/http/httptest" - "strings" "testing" - - "github.com/git-pkgs/registries/fetch" ) func TestCRANParseSourceFilename(t *testing.T) { @@ -56,68 +50,3 @@ func TestCRANParseBinaryFilename(t *testing.T) { } } } - -func TestCRANHandler_MacOSTwoSegmentIndexesAndMeta(t *testing.T) { - for _, path := range []string{ - "/bin/macosx/big-sur-arm64/contrib/4.4/PACKAGES", - "/bin/macosx/big-sur-x86_64/contrib/4.4/PACKAGES.gz", - "/bin/macosx/sonoma-arm64/contrib/4.5/PACKAGES.rds", - "/src/contrib/Meta/archive.rds", - "/src/contrib/Meta/current.rds", - } { - t.Run(path, func(t *testing.T) { - assertUpstreamProxied(t, "index", path, - func(proxy *Proxy, upstreamURL string) http.Handler { - h := &CRANHandler{proxy: proxy, upstreamURL: upstreamURL, proxyURL: "http://localhost"} - return h.Routes() - }, - ) - }) - } -} - -func TestCRANHandler_MacOSTwoSegmentBinaryDownload(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("fetched binary")), - ContentType: "application/x-gzip", - } - - h := NewCRANHandler(proxy, "http://localhost") - h.upstreamURL = "https://cran.r-project.org" - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz") - if err != nil { - t.Fatalf("request failed: %v", err) - } - _ = resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK) - } - want := "https://cran.r-project.org/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz" - if fetcher.fetchedURL != want { - t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) - } -} - -func TestCRANHandler_MacOSTwoSegmentBinaryCacheIdentity(t *testing.T) { - proxy, db, store, _ := setupTestProxy(t) - seedPackageWithPURL(t, db, store, "cran", "dplyr", "1.1.4_macosx-big-sur-arm64_4.4", "dplyr_1.1.4.tgz", "arm64 binary") - - h := NewCRANHandler(proxy, "http://localhost") - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - body, _ := io.ReadAll(resp.Body) - if string(body) != "arm64 binary" { - t.Errorf("body = %q, want %q", body, "arm64 binary") - } -} diff --git a/internal/handler/debian.go b/internal/handler/debian.go index 565f181..b767f6d 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -1,16 +1,10 @@ package handler import ( - "crypto/sha256" - "encoding/hex" "fmt" "net/http" "regexp" - "sort" "strings" - - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/purl" ) const ( @@ -18,64 +12,27 @@ const ( debMatchCount = 4 // full match + name + version + arch ) -// Packages are stored as pkg:deb/{name}, without a vendor and whichever -// configured archive they came from. Package rules may name a vendor the way -// PURLs usually do, pkg:deb/debian/curl or pkg:deb/ubuntu/curl; it is dropped, -// so the rule covers the package from every archive. -func init() { - retention.Default.Register(retention.Spec{Key: "deb", CanonicalPackage: debRetentionKey}) -} - -// debRetentionKey maps a package rule to the stored package PURL. A name with -// a slash, or a namespace with more than one segment, can never match a stored -// package and is refused. -func debRetentionKey(p *purl.PURL) (string, bool) { - if p == nil || p.Type != "deb" || strings.Contains(p.Namespace, "/") || strings.Contains(p.Name, "/") { - return "", false - } - canonical := purl.MakePURLString("deb", p.Name, "") - return canonical, canonical != "" -} - // DebianHandler handles APT/Debian repository protocol requests. // It proxies requests to upstream Debian/Ubuntu repositories and caches .deb packages. -// -// The main archive is served at /debian/. Additional archives are mounted at -// /debian/{repository}/ and the remaining path mirrors the upstream layout. type DebianHandler struct { - proxy *Proxy - upstreamURL string - proxyURL string - repositories map[string]string + proxy *Proxy + upstreamURL string + proxyURL string } // NewDebianHandler creates a new Debian/APT protocol handler. -// When repositories is empty, only the main archive is reachable. -func NewDebianHandler( - proxy *Proxy, - proxyURL string, - upstreamURL string, - repositories map[string]string, -) *DebianHandler { - if upstreamURL == "" { - upstreamURL = debianUpstream +func NewDebianHandler(proxy *Proxy, proxyURL string) *DebianHandler { + return &DebianHandler{ + proxy: proxy, + upstreamURL: debianUpstream, + proxyURL: strings.TrimSuffix(proxyURL, "/"), } - h := &DebianHandler{ - proxy: proxy, - upstreamURL: strings.TrimSuffix(upstreamURL, "/"), - proxyURL: strings.TrimSuffix(proxyURL, "/"), - repositories: make(map[string]string, len(repositories)), - } - for name, repositoryURL := range repositories { - h.repositories[name] = strings.TrimSuffix(repositoryURL, "/") - } - return h } // Routes returns the HTTP handler for Debian requests. // Mount this at /debian on your router. func (h *DebianHandler) Routes() http.Handler { - return rejectQueryChars(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet && r.Method != http.MethodHead { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return @@ -88,139 +45,61 @@ func (h *DebianHandler) Routes() http.Handler { return } - // Unlike the APK handler, an unconfigured first segment is not a 404: - // the main archive is unnamed and serves paths of its own at the root - // (README, indices/, project/), so an unknown name stays a - // main-archive path. - upstreamURL, repository, rest := h.upstreamURL, "", path - if !strings.HasPrefix(path, "pool/") && !strings.HasPrefix(path, "dists/") { - if name, tail, ok := strings.Cut(path, "/"); ok && tail != "" { - if named, found := h.repositories[name]; found { - upstreamURL, repository, rest = named, name, tail - } else if strings.HasPrefix(tail, "dists/") || strings.HasPrefix(tail, "pool/") { - // The main archive has no {name}/dists/ or {name}/pool/ of - // its own, so this is a misspelled repository rather than a - // main-archive path. Answering here keeps a typo from - // surfacing as the upstream's opaque HTML 404. - h.repositoryNotFound(w, name) - return - } - } - } - - // Route by filename rather than path prefix because repositories such as - // Docker's nest the pool under dists//. + // Route based on path type switch { - case isDebPackageFile(rest): - h.handlePackageDownload(w, r, repository, upstreamURL, rest) - case strings.HasPrefix(rest, "dists/"): - // Repository metadata - served through the metadata cache - h.handleMetadata(w, r, repository, upstreamURL, rest) + case strings.HasPrefix(path, "pool/"): + // Package downloads - cache these + h.handlePackageDownload(w, r, path) + case strings.HasPrefix(path, "dists/"): + // Repository metadata - proxy without caching (changes frequently) + h.handleMetadata(w, r, path) default: // Other files (like README, etc.) - proxy directly - h.proxyFile(w, r, upstreamURL, rest) + h.proxyFile(w, r, path) } - })) -} - -func isDebPackageFile(path string) bool { - return strings.HasSuffix(path, ".deb") -} - -// repositoryNotFound answers a request addressed to a repository that is not -// configured, naming the ones that are so a misspelling is self-evident. -func (h *DebianHandler) repositoryNotFound(w http.ResponseWriter, name string) { - if len(h.repositories) == 0 { - http.Error(w, - fmt.Sprintf("unknown debian repository %q: none are configured", name), - http.StatusNotFound) - return - } - - configured := make([]string, 0, len(h.repositories)) - for repository := range h.repositories { - configured = append(configured, repository) - } - sort.Strings(configured) - - http.Error(w, - fmt.Sprintf("unknown debian repository %q: configured repositories are %s", - name, strings.Join(configured, ", ")), - http.StatusNotFound) + }) } // handlePackageDownload fetches and caches .deb packages from the pool. // Pool path format: pool/{component}/{prefix}/{name}/{filename} // Example: pool/main/n/nginx/nginx_1.18.0-6_amd64.deb -func (h *DebianHandler) handlePackageDownload( - w http.ResponseWriter, - r *http.Request, - repository, upstreamURL, path string, -) { +func (h *DebianHandler) handlePackageDownload(w http.ResponseWriter, r *http.Request, path string) { // Parse the path to extract package info name, version, arch := h.parsePoolPath(path) if name == "" { // Can't parse, just proxy directly - h.proxyFile(w, r, upstreamURL, path) + h.proxyFile(w, r, path) return } filename := path[strings.LastIndex(path, "/")+1:] - downloadURL := fmt.Sprintf("%s/%s", upstreamURL, path) - - cacheFilename := h.artifactCacheFilename(repository, filename) + downloadURL := fmt.Sprintf("%s/%s", h.upstreamURL, path) h.proxy.Logger.Info("debian package download", - "repository", repository, "name", name, "version", version, "arch", arch, "filename", filename) result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), "deb", name, version, cacheFilename, downloadURL) + r.Context(), "deb", name, version, filename, downloadURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") + h.proxy.Logger.Error("failed to get debian package", "error", err) + http.Error(w, "failed to fetch package", http.StatusBadGateway) return } - w.Header().Set(headerContentType, "application/vnd.debian.binary-package") - ServeArtifactRequest(w, r, result) + w.Header().Set("Content-Type", "application/vnd.debian.binary-package") + ServeArtifact(w, result) } -// handleMetadata serves repository metadata files through the metadata cache, -// so they keep resolving while the upstream archive is unreachable. -func (h *DebianHandler) handleMetadata( - w http.ResponseWriter, - r *http.Request, - repository, upstreamURL, path string, -) { - cacheKey := h.metadataCacheKeyFor(repository, upstreamURL, path) - h.proxy.ProxyCached(w, r, fmt.Sprintf("%s/%s", upstreamURL, path), "debian", cacheKey, "*/*") -} - -// artifactCacheFilename scopes a package by repository, since the same -// filename can hold different bytes in different archives. The main archive -// keeps the unscoped filename its existing cache entries were stored under. -func (h *DebianHandler) artifactCacheFilename(repository, filename string) string { - if repository == "" { - return filename - } - return repository + "/" + filename -} - -// metadataCacheKeyFor returns the metadata cache key for a request. The main -// archive hashes its upstream URL and the path; a named repository hashes its -// identity as APKHandler.metadataCacheKey does. -func (h *DebianHandler) metadataCacheKeyFor(repository, upstreamURL, path string) string { - if repository == "" { - return hashedCacheKey(upstreamURL, path) - } - identity := repository + "\x00" + upstreamURL + "\x00" + path - digest := sha256.Sum256([]byte(identity)) - return hex.EncodeToString(digest[:]) +// handleMetadata proxies repository metadata files. +// These change frequently so we don't cache them. +func (h *DebianHandler) handleMetadata(w http.ResponseWriter, r *http.Request, path string) { + cacheKey := strings.ReplaceAll(path, "/", "_") + h.proxy.ProxyCached(w, r, fmt.Sprintf("%s/%s", h.upstreamURL, path), "debian", cacheKey, "*/*") } // proxyFile proxies any file directly without caching. -func (h *DebianHandler) proxyFile(w http.ResponseWriter, r *http.Request, upstreamURL, path string) { - h.proxy.ProxyFile(w, r, fmt.Sprintf("%s/%s", upstreamURL, path)) +func (h *DebianHandler) proxyFile(w http.ResponseWriter, r *http.Request, path string) { + h.proxy.ProxyFile(w, r, fmt.Sprintf("%s/%s", h.upstreamURL, path)) } // debPackagePattern matches .deb filenames to extract name, version, and arch. @@ -230,7 +109,12 @@ var debPackagePattern = regexp.MustCompile(`^(.+)_([^_]+)_([^_]+)\.deb$`) // parsePoolPath extracts package info from a pool path. // Example: pool/main/n/nginx/nginx_1.18.0-6_amd64.deb func (h *DebianHandler) parsePoolPath(path string) (name, version, arch string) { - filename := path[strings.LastIndex(path, "/")+1:] + // Get the filename + idx := strings.LastIndex(path, "/") + if idx < 0 { + return "", "", "" + } + filename := path[idx+1:] // Parse the filename matches := debPackagePattern.FindStringSubmatch(filename) diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index 54a72b6..dfdd326 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -1,15 +1,7 @@ package handler import ( - "fmt" - "net/http" - "net/http/httptest" - "strings" "testing" - "time" - - "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries/fetch" ) func TestDebianHandler_parsePoolPath(t *testing.T) { @@ -20,307 +12,12 @@ func TestDebianHandler_parsePoolPath(t *testing.T) { {"pool/main/libn/libncurses/libncurses6_6.2-1_amd64.deb", "libncurses6", "6.2-1", "amd64"}, {"pool/contrib/v/virtualbox/virtualbox_6.1.38-1_amd64.deb", "virtualbox", "6.1.38-1", "amd64"}, {"pool/main/g/git/git_2.39.2-1_arm64.deb", "git", "2.39.2-1", "arm64"}, - { - "pool/universe/n/nmap/nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb", - "nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", "amd64", - }, - {"pool/main/o/openssl/openssl_3.0.2-0ubuntu1.15~build1_amd64.deb", "openssl", "3.0.2-0ubuntu1.15~build1", "amd64"}, - {"nginx_1.18.0-6_amd64.deb", "nginx", "1.18.0-6", "amd64"}, {"invalid/path", "", "", ""}, {"pool/main/n/nginx/nginx.deb", "", "", ""}, }) } func TestDebianHandler_Routes(t *testing.T) { - h := NewDebianHandler(nil, "http://localhost:8080", "", nil) + h := NewDebianHandler(nil, "http://localhost:8080") assertRoutesBasics(t, h.Routes(), "/dists/stable/Release", "/pool/../../../etc/passwd") } - -// TestDebianHandler_LegacyCacheKeysUnchanged pins the artifact cache identity -// the main archive used before named repositories existed. Deployments carry -// warm caches across upgrades, so a changed key here silently discards them. -// Metadata is cheap to refetch; its keys are covered by -// TestMetadataCacheKeysUnambiguous. -func TestDebianHandler_LegacyCacheKeysUnchanged(t *testing.T) { - const ( - poolPath = "pool/main/h/hello/hello_2.10-3_amd64.deb" - distPath = "dists/trixie/InRelease" - ) - - h := NewDebianHandler(nil, "http://proxy.example", "https://archive.test", map[string]string{ - "security": "https://security.test", - }) - - // Artifact cache: the bare filename, with no repository prefix. - filename := poolPath[strings.LastIndex(poolPath, "/")+1:] - if got := h.artifactCacheFilename("", filename); got != "hello_2.10-3_amd64.deb" { - t.Errorf("legacy artifact cache filename = %q, want %q", got, "hello_2.10-3_amd64.deb") - } - - // A named repository must not reuse either identity. - if got := h.artifactCacheFilename("security", filename); got == "hello_2.10-3_amd64.deb" { - t.Error("named repository reused the legacy artifact cache filename") - } - if h.metadataCacheKeyFor("security", "https://security.test", distPath) == h.metadataCacheKeyFor("", h.upstreamURL, distPath) { - t.Error("named repository reused the main archive's metadata cache key") - } -} - -// TestDebianHandler_NamedRepositoryRouting checks that a named repository -// reaches its own archive while the main archive keeps serving /dists/ and -// /pool/ unchanged. -func TestDebianHandler_NamedRepositoryRouting(t *testing.T) { - mainRelease := "main archive InRelease" - mainArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/dists/trixie/InRelease" { - http.NotFound(w, r) - return - } - _, _ = fmt.Fprint(w, mainRelease) - })) - defer mainArchive.Close() - - securityRelease := "security archive InRelease" - securityArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/dists/trixie-security/InRelease" { - http.NotFound(w, r) - return - } - _, _ = fmt.Fprint(w, securityRelease) - })) - defer securityArchive.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = http.DefaultClient - - h := NewDebianHandler(proxy, "http://proxy.example", mainArchive.URL, map[string]string{ - "security": securityArchive.URL, - }) - - got := serveDebianRequest(h, "/security/dists/trixie-security/InRelease") - if got.Code != http.StatusOK || got.Body.String() != securityRelease { - t.Errorf("named repository: status = %d, body = %q, want 200 %q", - got.Code, got.Body.String(), securityRelease) - } - - got = serveDebianRequest(h, "/dists/trixie/InRelease") - if got.Code != http.StatusOK || got.Body.String() != mainRelease { - t.Errorf("main archive: status = %d, body = %q, want 200 %q", - got.Code, got.Body.String(), mainRelease) - } - - // An unconfigured name is not a repository, so it addresses the main - // archive as a plain path. - got = serveDebianRequest(h, "/unknown/dists/trixie/InRelease") - if got.Code == http.StatusOK { - t.Errorf("unknown repository: status = 200, want the main archive's 404") - } -} - -// TestDebianHandler_UnknownRepositoryReportsConfiguredNames covers a misspelled -// repository name. {name}/dists/ is never a main-archive path, so the handler -// answers directly rather than forwarding upstream, where the reply would be an -// opaque HTML 404 that does not mention the repository at all. -func TestDebianHandler_UnknownRepositoryReportsConfiguredNames(t *testing.T) { - var upstreamHits int - mainArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - upstreamHits++ - http.Error(w, "404 Not Found", http.StatusNotFound) - })) - defer mainArchive.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - - h := NewDebianHandler(proxy, "http://proxy.example", mainArchive.URL, map[string]string{ - "security": "http://security.example", - "ghcli": "http://ghcli.example", - }) - - got := serveDebianRequest(h, "/secuirty/dists/trixie-security/InRelease") - if got.Code != http.StatusNotFound { - t.Errorf("status = %d, want %d", got.Code, http.StatusNotFound) - } - body := got.Body.String() - if !strings.Contains(body, `"secuirty"`) { - t.Errorf("body = %q, want it to name the misspelled repository", body) - } - // Sorted, so the message is stable across map iteration order. - if !strings.Contains(body, "ghcli, security") { - t.Errorf("body = %q, want it to list the configured repositories", body) - } - if upstreamHits != 0 { - t.Errorf("upstream hits = %d, want 0: the handler should answer without forwarding", upstreamHits) - } - - // A root-level main-archive path still falls through, since the main - // archive really does serve files of its own there. - got = serveDebianRequest(h, "/project/trace/README") - if got.Code != http.StatusNotFound || upstreamHits != 1 { - t.Errorf("main-archive path: status = %d, upstream hits = %d, want 404 and 1", - got.Code, upstreamHits) - } -} - -// TestDebianHandler_MetadataCacheKeysDoNotCollideAcrossRepositories guards the -// hashed metadata cache key. The main archive's key replaces '/' with '_', so -// a repository named "security" serving dists/trixie/InRelease would otherwise -// collide with the main archive's own security_dists_trixie_InRelease entry, -// serving one archive's signed metadata to clients of the other. -func TestDebianHandler_MetadataCacheKeysDoNotCollideAcrossRepositories(t *testing.T) { - mainRelease := "main archive InRelease" - mainArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, mainRelease) - })) - defer mainArchive.Close() - - securityRelease := "security archive InRelease" - securityArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, securityRelease) - })) - defer securityArchive.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = http.DefaultClient - - h := NewDebianHandler(proxy, "http://proxy.example", mainArchive.URL, map[string]string{ - "security": securityArchive.URL, - }) - - // Main archive path whose separator-based key is "security_dists_...". - first := serveDebianRequest(h, "/dists/security/dists/trixie/InRelease") - if first.Code != http.StatusOK || first.Body.String() != mainRelease { - t.Fatalf("main archive: status = %d, body = %q, want 200 %q", - first.Code, first.Body.String(), mainRelease) - } - - // Served within the metadata TTL: a colliding key would return - // mainRelease here. - second := serveDebianRequest(h, "/security/dists/trixie/InRelease") - if second.Code != http.StatusOK { - t.Fatalf("named repository: status = %d, want 200: %s", second.Code, second.Body.String()) - } - if second.Body.String() != securityRelease { - t.Errorf("named repository served %q, want %q (cache key collision)", - second.Body.String(), securityRelease) - } -} - -// TestDebianHandler_PackageCacheScopedPerRepository checks that the same -// package filename in two archives does not resolve to one cached artifact. -func TestDebianHandler_PackageCacheScopedPerRepository(t *testing.T) { - const pkgPath = "/pool/main/h/hello/hello_2.10-3_amd64.deb" - - mainPkg := "main archive package bytes" - mainArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, mainPkg) - })) - defer mainArchive.Close() - - securityPkg := "security archive package bytes" - securityArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, securityPkg) - })) - defer securityArchive.Close() - - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(mainArchive.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewDebianHandler(proxy, "http://proxy.example", mainArchive.URL, map[string]string{ - "security": securityArchive.URL, - }) - - first := serveDebianRequest(h, pkgPath) - if first.Code != http.StatusOK || first.Body.String() != mainPkg { - t.Fatalf("main archive: status = %d, body = %q, want 200 %q", - first.Code, first.Body.String(), mainPkg) - } - - second := serveDebianRequest(h, "/security"+pkgPath) - if second.Code != http.StatusOK { - t.Fatalf("named repository: status = %d, want 200: %s", second.Code, second.Body.String()) - } - if second.Body.String() != securityPkg { - t.Errorf("named repository served %q, want %q (artifact cache collision)", - second.Body.String(), securityPkg) - } -} - -func TestDebianHandler_FlatArchivePackageIsCached(t *testing.T) { - const pkgBody = "docker-ce package bytes" - - var upstreamHits int - - archive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamHits++ - _, _ = fmt.Fprint(w, pkgBody) - })) - defer archive.Close() - - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(archive.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewDebianHandler(proxy, "http://proxy.example", "https://archive.test", map[string]string{ - "docker": archive.URL, - }) - - for _, path := range []string{ - "/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb", - "/docker/foo_1.0_amd64.deb", - } { - for attempt := 1; attempt <= 2; attempt++ { - w := serveDebianRequest(h, path) - if w.Code != http.StatusOK || w.Body.String() != pkgBody { - t.Fatalf("%s request %d: status = %d, body = %q, want 200 %q", - path, attempt, w.Code, w.Body.String(), pkgBody) - } - } - } - - if upstreamHits != 2 { - t.Errorf("upstream fetched %d times for four requests, want 2: the package is not cached", upstreamHits) - } -} - -func serveDebianRequest(h *DebianHandler, target string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - return w -} - -func TestDebRetentionKey(t *testing.T) { - tests := []struct { - key string - want string - ok bool - }{ - {"pkg:deb/curl", "pkg:deb/curl", true}, - {"pkg:deb/debian/curl", "pkg:deb/curl", true}, - {"pkg:deb/ubuntu/Curl", "pkg:deb/curl", true}, - {"pkg:deb/debian/libstdc++6", "pkg:deb/libstdc%2B%2B6", true}, - {"pkg:deb/debian/main/curl", "", false}, - {"pkg:deb/debian/pool%2Fcurl", "", false}, - {"pkg:apk/debian/curl", "", false}, - } - for _, tt := range tests { - p, err := purl.Parse(tt.key) - if err != nil { - t.Fatalf("Parse(%s): %v", tt.key, err) - } - got, ok := debRetentionKey(p) - if ok != tt.ok || (ok && got != tt.want) { - t.Errorf("debRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) - } - } - if _, ok := debRetentionKey(nil); ok { - t.Error("debRetentionKey(nil) accepted") - } -} diff --git a/internal/handler/denylist.go b/internal/handler/denylist.go deleted file mode 100644 index a03f03e..0000000 --- a/internal/handler/denylist.go +++ /dev/null @@ -1,8 +0,0 @@ -package handler - -func (p *Proxy) versionDenied(ecosystem, name, version string) bool { - if p.Denylist == nil { - return false - } - return p.Denylist.Denied(canonicalVersionPURL(ecosystem, name, version)) -} diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go deleted file mode 100644 index 663d0c3..0000000 --- a/internal/handler/denylist_test.go +++ /dev/null @@ -1,427 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/proxy/internal/denylist" - "github.com/git-pkgs/registries/fetch" -) - -func setTestDenylist(t testing.TB, p *Proxy, packages ...string) { - t.Helper() - var err error - p.Denylist, err = denylist.New(packages) - if err != nil { - t.Fatal(err) - } -} - -func TestDenylistBlocksArtifactAccess(t *testing.T) { - for _, tc := range []struct{ cached, direct bool }{{false, false}, {false, true}, {true, false}, {true, true}} { - cached, direct := tc.cached, tc.direct - t.Run(fmt.Sprintf("cached=%t/direct=%t", cached, direct), func(t *testing.T) { - p, db, store, _ := setupTestProxy(t) - if cached { - seedPackage(t, db, store, "npm", "demo", "1.0.0", "demo.tgz", "content") - } - p.DirectServe = direct - store.signedURL = "https://storage.example/demo" - setTestDenylist(t, p, "pkg:npm/demo@1.0.0") - ctx := context.Background() - for _, get := range []func() (*CacheResult, error){ - func() (*CacheResult, error) { return p.GetOrFetchArtifact(ctx, "npm", "demo", "1.0.0", "demo.tgz") }, - func() (*CacheResult, error) { return p.GetCachedArtifact(ctx, "npm", "demo", "1.0.0", "demo.tgz") }, - func() (*CacheResult, error) { - return p.GetOrFetchArtifactFromURL(ctx, "npm", "demo", "1.0.0", "demo.tgz", "https://upstream.invalid/demo") - }, - func() (*CacheResult, error) { - return p.GetOrFetchArtifactFromURLWithHeaders(ctx, "npm", "demo", "1.0.0", "demo.tgz", "https://upstream.invalid/demo", nil) - }, - func() (*CacheResult, error) { - return p.GetOrFetchArtifactFromURLWithDigest(ctx, "npm", "demo", "1.0.0", "demo.tgz", "https://upstream.invalid/demo", "sha256:abc") - }, - } { - result, err := get() - if result != nil || !errors.Is(err, ErrVersionDenied) { - t.Fatalf("result=%+v err=%v; want denial", result, err) - } - w := httptest.NewRecorder() - p.serveArtifactError(w, err, "fetch failed") - if w.Code != http.StatusForbidden { - t.Fatalf("status = %d", w.Code) - } - } - // Denying does not purge the cache. Removing the policy restores it. - p.Denylist = nil - result, err := p.GetCachedArtifact(ctx, "npm", "demo", "1.0.0", "demo.tgz") - if err != nil || (result != nil) != cached { - t.Fatalf("cache changed: result=%+v err=%v", result, err) - } - if result != nil && result.Reader != nil { - _ = result.Reader.Close() - } - }) - } -} - -func TestNPMDenylistMetadata(t *testing.T) { - p := testProxy() - setTestDenylist(t, p, "pkg:npm/@scope/demo@2.0.0") - h := NewNPMHandler(p, "https://proxy.example", "") - for _, times := range []string{"", `,"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z"}`} { - body := `{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0-beta.1":{}},"dist-tags":{"latest":"2.0.0","beta":"2.0.0","next":"3.0.0-beta.1"}` + times + `}` - out, _, err := h.rewriteMetadataKeeping("@scope/demo", []byte(body)) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(out), "2.0.0") || !strings.Contains(string(out), `"latest":"1.0.0"`) || !strings.Contains(string(out), `"next":"3.0.0-beta.1"`) { - t.Fatalf("incorrect metadata: %s", out) - } - } - out, _, err := h.rewriteMetadataKeeping("@scope/demo", []byte(`{"versions":{"2.0.0":{}},"dist-tags":{"latest":"2.0.0"}}`)) - if err != nil || strings.Contains(string(out), "2.0.0") { - t.Fatalf("all denied: %s, %v", out, err) - } -} - -func TestCargoDenylistWithoutTimestamps(t *testing.T) { - p := testProxy() - setTestDenylist(t, p, "pkg:cargo/demo@1.0.0") - h := &CargoHandler{proxy: p} - input := "{\"name\":\"demo\",\"vers\":\"1.0.0\"}\n{\"name\":\"demo\",\"vers\":\"2.0.0\"}\n" - for _, cd := range []CooldownPolicy{nil, &cooldown.Config{Default: "3d"}} { - p.Cooldown = cd - w := httptest.NewRecorder() - h.applyCooldownFiltering(w, []byte(input)) - if w.Body.String() != "{\"name\":\"demo\",\"vers\":\"2.0.0\"}\n" { - t.Fatalf("index: %s", w.Body.String()) - } - } -} - -func TestPyPIDenylistSimpleRepresentations(t *testing.T) { - for _, tc := range []struct{ name, contentType, body string }{ - {"html", "text/html", `downloaddemo-2.0.0.tar.gz`}, - {"json", pypiSimpleJSON, `{"meta":{"api-version":"1.0"},"files":[{"filename":"demo-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/a/b/demo-1.0.0.tar.gz"},{"filename":"demo-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/a/b/demo-2.0.0.tar.gz"}]}`}, - } { - t.Run(tc.name, func(t *testing.T) { - h, p := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - if r.URL.Path != "/simple/demo/" { - t.Fatalf("denylist must not need timestamp metadata: %s", r.URL.Path) - } - return pypiHTTPResponse(r, tc.contentType, tc.body), nil - }) - p.CacheMetadata = true - p.MetadataTTL = time.Hour - // Seed raw metadata before configuring the denylist. - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/simple/demo/", nil)) - setTestDenylist(t, p, "pkg:pypi/Demo@1.0.0") - p.HTTPClient = &http.Client{Transport: pypiRoundTripFunc(func(*http.Request) (*http.Response, error) { - return nil, errors.New("offline") - })} - for _, ttl := range []time.Duration{time.Hour, 0} { - p.MetadataTTL = ttl - w = httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/simple/demo/", nil)) - if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "1.0.0") || !strings.Contains(w.Body.String(), "2.0.0") { - t.Fatalf("ttl=%s status=%d body=%s", ttl, w.Code, w.Body.String()) - } - } - }) - } -} - -func TestPyPIDenylistJSONMetadata(t *testing.T) { - p := testProxy() - setTestDenylist(t, p, "pkg:pypi/demo@1.0.0") - h := NewPyPIHandlerWithUpstreams(p, "https://proxy.example", "", "") - body := `{"info":{"name":"Demo","version":"1.0.0"},"releases":{"1.0.0":[{"url":"https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz"}],"2.0.0":[]},"urls":[{"url":"https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz"}]}` - out, _, err := h.rewriteJSONMetadata([]byte(body)) - if err != nil { - t.Fatal(err) - } - var metadata map[string]any - if err := json.Unmarshal(out, &metadata); err != nil { - t.Fatal(err) - } - if _, exists := metadata["releases"].(map[string]any)["1.0.0"]; exists || len(metadata["urls"].([]any)) != 0 { - t.Fatalf("denied release survived: %s", out) - } - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/demo/1.0.0/json", nil)) - if w.Code != http.StatusNotFound { - t.Fatalf("denied version metadata status=%d", w.Code) - } -} - -func TestDenylistDownloadRoutes(t *testing.T) { - for _, tc := range []struct { - ecosystem, path, filename string - handler func(*Proxy) http.Handler - }{ - {"npm", "/demo/-/demo-1.0.0.tgz", "demo-1.0.0.tgz", func(p *Proxy) http.Handler { return NewNPMHandler(p, "http://proxy.test", "").Routes() }}, - {"pypi", "/packages/a/b/demo-1.0.0.tar.gz", "demo-1.0.0.tar.gz", func(p *Proxy) http.Handler { - return NewPyPIHandlerWithUpstreams(p, "http://proxy.test", "", "").Routes() - }}, - {"cargo", "/crates/demo/1.0.0/download", "demo-1.0.0.crate", func(p *Proxy) http.Handler { return NewCargoHandler(p, "http://proxy.test", "", "").Routes() }}, - {"deb", "/pool/main/d/demo/demo_1.0.0_amd64.deb", "demo_1.0.0_amd64.deb", func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://proxy.test", "", nil).Routes() }}, - } { - t.Run(tc.ecosystem, func(t *testing.T) { - p, db, store, _ := setupTestProxy(t) - seedPackage(t, db, store, tc.ecosystem, "demo", "1.0.0", tc.filename, "cached content") - setTestDenylist(t, p, "pkg:"+tc.ecosystem+"/demo@1.0.0") - w := httptest.NewRecorder() - tc.handler(p).ServeHTTP(w, httptest.NewRequest(http.MethodGet, tc.path, nil)) - if w.Code != http.StatusForbidden || !strings.Contains(w.Body.String(), "denylist") { - t.Fatalf("denied download status=%d body=%s", w.Code, w.Body.String()) - } - // An allowed version remains readable while the policy is enabled. - filename := strings.ReplaceAll(tc.filename, "1.0.0", "2.0.0") - seedPackage(t, db, store, tc.ecosystem, "demo", "2.0.0", filename, "allowed content") - result, err := p.GetCachedArtifact(context.Background(), tc.ecosystem, "demo", "2.0.0", filename) - if err != nil || result == nil { - t.Fatalf("allowed download: result=%v err=%v", result, err) - } - defer func() { _ = result.Reader.Close() }() - body, err := io.ReadAll(result.Reader) - if err != nil || string(body) != "allowed content" { - t.Fatalf("allowed body=%s err=%v", body, err) - } - }) - } -} - -func TestDenylistAndCooldownCompose(t *testing.T) { - p := testProxy() - p.Cooldown = &cooldown.Config{Default: "3d"} - setTestDenylist(t, p, "pkg:npm/demo@1.0.0") - h := NewNPMHandler(p, "https://proxy.test", "") - body := fmt.Sprintf(`{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0":{}},"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z","3.0.0":%q},"dist-tags":{"latest":"3.0.0"}}`, time.Now().UTC().Format(time.RFC3339)) - out, _, err := h.rewriteMetadataKeeping("demo", []byte(body)) - if err != nil || strings.Contains(string(out), "1.0.0") || strings.Contains(string(out), "3.0.0") || !strings.Contains(string(out), `"latest":"2.0.0"`) { - t.Fatalf("combined policy: %s, %v", out, err) - } -} - -func TestCargoDenylistLargeEntry(t *testing.T) { - p := testProxy() - setTestDenylist(t, p, "pkg:cargo/demo@1.0.0") - h := &CargoHandler{proxy: p} - line := fmt.Sprintf("{\"name\":\"demo\",\"vers\":\"2.0.0\",\"extra\":%q}\n", strings.Repeat("x", 70<<10)) - w := httptest.NewRecorder() - h.applyCooldownFiltering(w, []byte(line)) - if w.Body.String() != line { - t.Fatal("large allowed index entry was truncated") - } -} - -func TestDenylistMalformedMetadataFailsClosed(t *testing.T) { - for _, tc := range []struct { - name, path, body, contentType string - handler func(*Proxy) http.Handler - }{ - {"npm", "/demo", `{"versions":`, contentTypeJSON, func(p *Proxy) http.Handler { return NewNPMHandler(p, "https://proxy.test", "").Routes() }}, - {"pypi-json", "/pypi/demo/json", `{"releases":`, contentTypeJSON, func(p *Proxy) http.Handler { - return NewPyPIHandlerWithUpstreams(p, "https://proxy.test", "", "").Routes() - }}, - {"pypi-simple", "/simple/demo/", `{"files":{}}`, pypiSimpleJSON, func(p *Proxy) http.Handler { - return NewPyPIHandlerWithUpstreams(p, "https://proxy.test", "", "").Routes() - }}, - } { - t.Run(tc.name, func(t *testing.T) { - p := testProxy() - setTestDenylist(t, p, "pkg:npm/demo@1.0.0", "pkg:pypi/demo@1.0.0") - p.HTTPClient = &http.Client{Transport: pypiRoundTripFunc(func(r *http.Request) (*http.Response, error) { - return pypiHTTPResponse(r, tc.contentType, tc.body), nil - })} - w := httptest.NewRecorder() - tc.handler(p).ServeHTTP(w, httptest.NewRequest(http.MethodGet, tc.path, nil)) - if w.Code != http.StatusBadGateway { - t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) - } - }) - } -} - -// Denylist entries must match however the ecosystem spells the name or -// version in download URLs. -func TestDenylistMatchesEcosystemSpellings(t *testing.T) { - pypi := func(p *Proxy) http.Handler { - return NewPyPIHandlerWithUpstreams(p, "http://proxy.test", "", "").Routes() - } - for _, tc := range []struct { - name, entry, path string - ecosystem, pkg string - version, filename string - handler func(*Proxy) http.Handler - }{ - {"pypi wheel", "pkg:pypi/zope.interface@6.0", "/packages/a/b/zope_interface-6.0-py3-none-any.whl", - "pypi", "zope_interface", "6.0", "zope_interface-6.0-py3-none-any.whl", pypi}, - {"pypi dotted sdist", "pkg:pypi/zope-interface@6.0", "/packages/a/b/zope.interface-6.0.tar.gz", - "pypi", "zope.interface", "6.0", "zope.interface-6.0.tar.gz", pypi}, - {"nuget", "pkg:nuget/Newtonsoft.Json@13.0.1-Beta", "/v3-flatcontainer/newtonsoft.json/13.0.1-beta/newtonsoft.json.13.0.1-beta.nupkg", - "nuget", "newtonsoft.json", "13.0.1-beta", "newtonsoft.json.13.0.1-beta.nupkg", - func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://proxy.test").Routes() }}, - {"gem platform", "pkg:gem/nokogiri@1.16.0", "/gems/nokogiri-1.16.0-x86_64-linux.gem", - "gem", "nokogiri", "1.16.0-x86_64-linux", "nokogiri-1.16.0-x86_64-linux.gem", - func(p *Proxy) http.Handler { return NewGemHandler(p, "http://proxy.test").Routes() }}, - {"conda any channel", "pkg:conda/numpy@1.26.4", "/conda-forge/linux-64/numpy-1.26.4-py311_0.conda", - "conda", "conda-forge/numpy", "1.26.4", "numpy-1.26.4-py311_0.conda", - func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://proxy.test").Routes() }}, - {"conda channel qualifier", "pkg:conda/numpy@1.26.4?channel=conda-forge", "/conda-forge/linux-64/numpy-1.26.4-py311_0.conda", - "conda", "conda-forge/numpy", "1.26.4", "numpy-1.26.4-py311_0.conda", - func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://proxy.test").Routes() }}, - {"gem name with digit segment", "pkg:gem/devise-2fa@1.0.0", "/gems/devise-2fa-1.0.0.gem", - "gem", "devise-2fa", "1.0.0", "devise-2fa-1.0.0.gem", - func(p *Proxy) http.Handler { return NewGemHandler(p, "http://proxy.test").Routes() }}, - {"go case-escaped version", "pkg:golang/github.com/example/mod@v1.0.0-RC1", "/github.com/example/mod/@v/v1.0.0-!r!c1.zip", - "golang", "github.com/example/mod", "v1.0.0-!r!c1", "mod@v1.0.0-!r!c1.zip", - func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, - {"go lowercase rule", "pkg:golang/github.com/example/mod@v1.0.0", "/github.com/!example/mod/@v/v1.0.0.zip", - "golang", "github.com/Example/mod", "v1.0.0", goZipFilename("github.com/Example/mod", "v1.0.0"), - func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, - {"go mixed-case rule", "pkg:golang/github.com/Example/mod@v1.0.0", "/github.com/example/mod/@v/v1.0.0.zip", - "golang", "github.com/example/mod", "v1.0.0", "mod@v1.0.0.zip", - func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, - {"oci repository path", "pkg:oci/library/alpine@" + ociTestDigest, "/library/alpine/blobs/" + ociTestDigest, - "oci", "library/alpine", ociTestDigest, ociTestDigest, - func(p *Proxy) http.Handler { - return NewContainerHandlerWithRegistry(p, "http://proxy.test", "https://registry.invalid").Routes() - }}, - } { - t.Run(tc.name, func(t *testing.T) { - // The seeded artifact would be served if the entry didn't match. - p, db, store, fetcher := setupTestProxy(t) - fetcher.fetchErr = errors.New("offline") - seedPackage(t, db, store, tc.ecosystem, tc.pkg, tc.version, tc.filename, "cached content") - setTestDenylist(t, p, tc.entry) - w := httptest.NewRecorder() - tc.handler(p).ServeHTTP(w, httptest.NewRequest(http.MethodGet, tc.path, nil)) - if w.Code != http.StatusForbidden { - t.Fatalf("denied download status=%d body=%s", w.Code, w.Body.String()) - } - }) - } -} - -const ociTestDigest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" - -// Packagist keeps a tag's leading v in the version it lists, and so in the -// download URL the proxy hands out. -func TestComposerDenylistMatchesTagPrefix(t *testing.T) { - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/p2/symfony/console.json": - _, _ = fmt.Fprintf(w, `{"packages":{"symfony/console":[{"version":"v6.1.7","version_normalized":"6.1.7.0","dist":{"url":%q,"type":"zip"}}]}}`, - srv.URL+"/archives/console.zip") - case "/archives/console.zip": - _, _ = io.WriteString(w, "archive") - default: - http.NotFound(w, r) - } - })) - t.Cleanup(srv.Close) - p, _, _, _ := setupTestProxy(t) - p.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) - setTestDenylist(t, p, "pkg:composer/symfony/console@6.1.7") - h := NewComposerHandlerWithUpstreams(p, "http://proxy.test", srv.URL, srv.URL).Routes() - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/symfony/console/v6.1.7/console.zip", nil)) - if w.Code != http.StatusForbidden { - t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) - } -} - -// Swift archives are cached under pkg:generic/swift-registry/{scope}.{name}, -// which is also what streaming checks. -func TestSwiftDenylistBlocksArchive(t *testing.T) { - for _, stream := range []bool{false, true} { - for _, entry := range []string{"pkg:swift/Apple/Example@1.2.3", "pkg:generic/swift-registry/apple.example@1.2.3"} { - t.Run(fmt.Sprintf("stream=%t/%s", stream, entry), func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":"`+strings.Repeat("0", 64)+`"}]}`) - })) - t.Cleanup(upstream.Close) - p, _, _, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("archive")), Size: 7} - p.StreamArtifacts = stream - setTestDenylist(t, p, entry) - req := httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil) - req.Header.Set("Accept", swiftAcceptArchive) - w := httptest.NewRecorder() - NewSwiftHandler(p, "http://proxy.test", upstream.URL).Routes().ServeHTTP(w, req) - if w.Code != http.StatusForbidden || fetcher.fetchCalled { - t.Fatalf("status=%d fetched=%t body=%s", w.Code, fetcher.fetchCalled, w.Body.String()) - } - }) - } - } -} - -// A denied digest is refused as a manifest as well as a blob, whether the -// client names it or a tag resolves to it. -func TestContainerDenylistBlocksManifests(t *testing.T) { - manifest := `{"schemaVersion":2}` - digest := sha256Digest([]byte(manifest)) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set(headerContentType, "application/vnd.oci.image.manifest.v1+json") - w.Header().Set("Docker-Content-Digest", digest) - _, _ = io.WriteString(w, manifest) - })) - t.Cleanup(upstream.Close) - for _, tc := range []struct{ entry, reference string }{ - {"pkg:oci/library/alpine@" + digest, digest}, - {"pkg:oci/library/alpine@" + digest, "3.19"}, - {"pkg:oci/library/alpine@3.19", "3.19"}, - } { - t.Run(tc.entry+"/"+tc.reference, func(t *testing.T) { - p := testProxy() - p.HTTPClient = upstream.Client() - h := NewContainerHandlerWithRegistry(p, "http://proxy.test", upstream.URL).Routes() - // Cache the manifest first, then deny it. - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/library/alpine/manifests/"+tc.reference, nil)) - if w.Code != http.StatusOK { - t.Fatalf("allowed status=%d", w.Code) - } - setTestDenylist(t, p, tc.entry) - for _, method := range []string{http.MethodGet, http.MethodHead} { - w = httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(method, "/library/alpine/manifests/"+tc.reference, nil)) - if w.Code != http.StatusForbidden { - t.Fatalf("%s status=%d body=%s", method, w.Code, w.Body.String()) - } - } - w = httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/library/alpine/manifests/other", nil)) - if strings.HasPrefix(tc.entry, "pkg:oci/library/alpine@3.19") && w.Code != http.StatusOK { - t.Fatalf("other tag status=%d", w.Code) - } - }) - } -} - -func TestPyPIDenylistSimpleMatchesSeparators(t *testing.T) { - body := `xy` - h, p := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - return pypiHTTPResponse(r, "text/html", body), nil - }) - setTestDenylist(t, p, "pkg:pypi/zope.interface@6.0") - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/simple/zope-interface/", nil)) - if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "6.0") || !strings.Contains(w.Body.String(), "7.0") { - t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) - } -} diff --git a/internal/handler/detached_fetch_test.go b/internal/handler/detached_fetch_test.go deleted file mode 100644 index 554761d..0000000 --- a/internal/handler/detached_fetch_test.go +++ /dev/null @@ -1,165 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "net/http" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/registries/fetch" -) - -// gatedFetcher holds every fetch until release is closed, and gives up early -// when the fetch context is cancelled, the way a real HTTP fetch does. -type gatedFetcher struct { - calls atomic.Int64 - content string - entered chan struct{} - release chan struct{} -} - -func newGatedFetcher(content string) *gatedFetcher { - return &gatedFetcher{content: content, entered: make(chan struct{}, 1), release: make(chan struct{})} -} - -func (f *gatedFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { - return f.FetchWithHeaders(ctx, url, nil) -} - -func (f *gatedFetcher) FetchWithHeaders(ctx context.Context, _ string, _ http.Header) (*fetch.Artifact, error) { - f.calls.Add(1) - select { - case f.entered <- struct{}{}: - default: - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-f.release: - } - return &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader(f.content)), - ContentType: "application/octet-stream", - }, nil -} - -func (f *gatedFetcher) Head(context.Context, string) (int64, string, error) { - return 0, "", nil -} - -const detachedFetchURL = "https://geo.mirror.pkgbuild.com/extra/os/x86_64/big-1.0-1-x86_64.pkg.tar.zst" - -// startCancelledFetch starts a fetch under ctx, waits until it reached the -// upstream, cancels ctx, lets the upstream answer, and returns the caller's -// error. -func startCancelledFetch(t *testing.T, proxy *Proxy, fetcher *gatedFetcher, ctx context.Context) error { - t.Helper() - ctx, cancel := context.WithCancel(ctx) - done := make(chan error, 1) - go func() { - res, err := proxy.GetOrFetchArtifactFromURL(ctx, "arch", "big", "1.0-1", "big-1.0-1-x86_64.pkg.tar.zst", detachedFetchURL) - drain(res) - done <- err - }() - - select { - case <-fetcher.entered: - case <-time.After(5 * time.Second): - t.Fatal("fetch never reached the upstream") - } - cancel() - // Give a non-detached fetch the chance to observe the cancellation before - // the upstream answers. - time.Sleep(50 * time.Millisecond) - close(fetcher.release) - - select { - case err := <-done: - return err - case <-time.After(5 * time.Second): - t.Fatal("fetch did not return") - return nil - } -} - -// TestDetachedFetch_ClientCancelStillCaches covers a client that gives up on a -// slow cache miss. pacman aborts after 10 s without data; if that cancelled -// the upstream fetch, every retry would start from zero and a large package -// would never reach the cache. -func TestDetachedFetch_ClientCancelStillCaches(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - fetcher := newGatedFetcher("package bytes") - proxy.Fetcher = fetcher - - _ = startCancelledFetch(t, proxy, fetcher, DetachUpstreamFetch(context.Background())) - - res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "arch", "big", "1.0-1", "big-1.0-1-x86_64.pkg.tar.zst", detachedFetchURL) - if err != nil { - t.Fatalf("follow-up request: %v", err) - } - body, _ := io.ReadAll(res.Reader) - _ = res.Reader.Close() - if !res.Cached { - t.Error("follow-up was not a cache hit: the cancelled client aborted the fetch") - } - if string(body) != "package bytes" { - t.Errorf("body = %q, want %q", body, "package bytes") - } - if got := fetcher.calls.Load(); got != 1 { - t.Errorf("upstream fetches = %d, want 1", got) - } -} - -// TestDetachedFetch_UnmarkedContextStillCancels keeps mirror jobs cancellable: -// without the marker, cancelling the context aborts the fetch and nothing is -// cached. -func TestDetachedFetch_UnmarkedContextStillCancels(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - fetcher := newGatedFetcher("package bytes") - proxy.Fetcher = fetcher - - err := startCancelledFetch(t, proxy, fetcher, context.Background()) - if !errors.Is(err, context.Canceled) { - t.Fatalf("error = %v, want context.Canceled", err) - } - - cached, err := proxy.GetCachedArtifact(context.Background(), "arch", "big", "1.0-1", "big-1.0-1-x86_64.pkg.tar.zst") - if err != nil { - t.Fatalf("cache lookup: %v", err) - } - if cached != nil { - _ = cached.Reader.Close() - t.Error("cancelled undetached fetch was still committed to the cache") - } -} - -// TestDetachedFetch_CapFallsBackToClientContext checks the cap on detached -// fetches: with every slot taken, a cancelled client cancels its fetch again, -// as it did before fetches were detached. -func TestDetachedFetch_CapFallsBackToClientContext(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - fetcher := newGatedFetcher("package bytes") - proxy.Fetcher = fetcher - - for range maxDetachedFetches { - if !proxy.acquireDetachSlot() { - t.Fatal("could not take a free detach slot") - } - } - err := startCancelledFetch(t, proxy, fetcher, DetachUpstreamFetch(context.Background())) - if !errors.Is(err, context.Canceled) { - t.Fatalf("error = %v, want context.Canceled once the detach slots are used up", err) - } - - for range maxDetachedFetches { - proxy.releaseDetachSlot() - } - if !proxy.acquireDetachSlot() { - t.Error("slots not available again after release") - } -} diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go deleted file mode 100644 index 4ede9a8..0000000 --- a/internal/handler/download_cooldown.go +++ /dev/null @@ -1,159 +0,0 @@ -package handler - -import ( - "net/http" - "sync" - "time" - - "golang.org/x/sync/singleflight" -) - -// publishTimeLookup reads a version's publish time from upstream metadata. ok -// is false when the metadata has no usable time for the version. withholdUnknown -// marks a listed version missing from stale publish-time metadata. -type publishTimeLookup func() (published time.Time, ok, withholdUnknown bool, err error) - -// downloadVersion names the version an artifact download is for, so its -// cooldown can be checked before the artifact is served. -type downloadVersion struct { - ecosystem string - // policyName is the package name the cooldown policy is keyed by, - // canonicalPackagePURL(ecosystem, policyName). - policyName string - // storedName and version name the versions row the publish time is kept - // in, the same names the artifact is cached under. An empty storedName - // keeps no time, and every check asks lookup. - storedName string - version string - cacheName string - filename string - lookup publishTimeLookup -} - -// withholdInCooldown reports whether a download is inside the cooldown -// window, or a lookup failure prevents an uncached download. -func (p *Proxy) withholdInCooldown(w http.ResponseWriter, r *http.Request, d downloadVersion) bool { - withheld, err := p.versionInCooldown(d) - if err != nil { - if !p.serveCachedDespiteCooldownError(w, r, d.ecosystem, d.cacheName, d.version, d.filename, err) { - p.writeCooldownLookupError(w, d.ecosystem, d.policyName, err) - } - return true - } - if !withheld { - return false - } - p.Logger.Info("cooldown: withholding "+d.ecosystem+" download", - "name", d.policyName, "version", d.version) - http.Error(w, ErrVersionNotFound.Error(), http.StatusNotFound) - return true -} - -func (p *Proxy) versionInCooldown(d downloadVersion) (bool, error) { - policyKey := canonicalPackagePURL(d.ecosystem, d.policyName) - if !p.cooldownApplies(d.ecosystem, d.policyName) { - return false, nil - } - - var versionPURL string - if d.storedName != "" { - versionPURL = canonicalVersionPURL(d.ecosystem, d.storedName, d.version) - if ver, err := p.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return !p.cooldown().IsAllowed(d.ecosystem, policyKey, ver.PublishedAt.Time), nil - } - } - - published, ok, withholdUnknown, err := d.lookup() - if err != nil { - return false, err - } - if !ok { - return withholdUnknown, nil - } - - if versionPURL != "" { - if err := p.DB.SetVersionPublishedAt(versionPURL, canonicalPackagePURL(d.ecosystem, d.storedName), published); err != nil { - p.Logger.Warn("cooldown: could not store publish time", - "ecosystem", d.ecosystem, "name", d.policyName, "version", d.version, "error", err) - } - } - - return !p.cooldown().IsAllowed(d.ecosystem, policyKey, published), nil -} - -// maxPublishIndexes bounds how many documents publishIndexCache keeps an -// index for. -const maxPublishIndexes = 64 - -// publishIndexCache keeps, for each large metadata document, the publish -// times a download check reads from it, so a document is decompressed and -// parsed once per version from upstream rather than on every download. -// Concurrent checks for a document not indexed yet share one parse. -type publishIndexCache struct { - mu sync.Mutex - entries map[string]publishIndexEntry - group singleflight.Group -} - -type publishIndexEntry struct { - digest string - index any - expires time.Time -} - -// publishIndex reuses parsed documents by digest, or for a minute without one. -func publishIndex[V any](p *Proxy, key string, load func() metadataResult, build func(metadataResult) (V, error)) (V, error) { //nolint:ireturn // Supports timestamp maps and Helm lookups. - c := &p.sharedProxy().publishIndexes - fresh := func() (V, bool) { - c.mu.Lock() - defer c.mu.Unlock() - if e, ok := c.entries[key]; ok && !e.expires.IsZero() && p.clock().Before(e.expires) { - return e.index.(V), true - } - var zero V - return zero, false - } - if index, ok := fresh(); ok { - return index, nil - } - v, err, _ := c.group.Do(key, func() (any, error) { - if index, ok := fresh(); ok { - return index, nil - } - res := load() - if res.err == nil && res.digest != "" { - c.mu.Lock() - entry, ok := c.entries[key] - c.mu.Unlock() - if ok && entry.digest == res.digest { - return entry.index, nil - } - } - index, err := build(res) - if err != nil { - return nil, err - } - c.mu.Lock() - defer c.mu.Unlock() - if c.entries == nil { - c.entries = make(map[string]publishIndexEntry) - } - if _, ok := c.entries[key]; !ok && len(c.entries) >= maxPublishIndexes { - for evict := range c.entries { - delete(c.entries, evict) - break - } - } - entry := publishIndexEntry{digest: res.digest, index: index} - if res.digest == "" || !p.CacheMetadata { - entry.expires = p.clock().Add(time.Minute) - } - c.entries[key] = entry - return index, nil - }) - if err != nil { - var zero V - return zero, err - } - return v.(V), nil -} diff --git a/internal/handler/download_cooldown_cache_test.go b/internal/handler/download_cooldown_cache_test.go deleted file mode 100644 index d51b862..0000000 --- a/internal/handler/download_cooldown_cache_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package handler - -import ( - "errors" - "fmt" - "net/http" - "net/http/httptest" - "testing" - "time" -) - -func TestGemDownloadSharesPublishTimeMetadata(t *testing.T) { - u, proxy, h := gemCacheSetup(t) - wantStatus(t, serveGet(t, h, gemInfoPath), http.StatusOK, "") - seedPackage(t, proxy.DB, proxy.Storage.(*mockStorage), "gem", "testgem", "1.0.0", "testgem-1.0.0.gem", "gem bytes") - wantStatus(t, serveGet(t, h, "/gems/testgem-1.0.0.gem"), http.StatusOK, "") - if n, _ := u.counts(gemVersionsPath); n != 1 { - t.Fatalf("versions API fetched %d times, want 1", n) - } -} - -type downloadPublishTimeCase struct { - name string - down bool - absent bool - untimed bool - refresh bool - wantStatus int -} - -func TestDownloadCooldownRefreshesMissingPublishTimes(t *testing.T) { - for _, c := range downloadCooldownCases() { - if c.ecosystem != "gem" && c.ecosystem != "hex" { - continue - } - for _, scenario := range []downloadPublishTimeCase{ - {name: "refreshed times", refresh: true, wantStatus: http.StatusNotFound}, - {name: "stale times and listed version", down: true, wantStatus: http.StatusNotFound}, - {name: "stale times and absent version", down: true, absent: true, wantStatus: http.StatusOK}, - {name: "fresh times without version", wantStatus: http.StatusOK}, - {name: "stale times with untimed version", down: true, untimed: true, wantStatus: http.StatusOK}, - } { - for _, cached := range []bool{false, true} { - if !cached && scenario.wantStatus == http.StatusOK { - continue - } - t.Run(fmt.Sprintf("%s/%s/cached=%t", c.ecosystem, scenario.name, cached), func(t *testing.T) { - assertDownloadPublishTime(t, c, scenario, cached) - }) - } - } - } -} - -func assertDownloadPublishTime(t *testing.T, c downloadCooldownCase, scenario downloadPublishTimeCase, cached bool) { - t.Helper() - u := newCooldownCountingUpstream(t) - proxy := cooldownCacheProxy(t, u) - fetcher := proxy.Fetcher.(*mockFetcher) - fetcher.fetchErr = errors.New("unchecked artifact download") - h := c.handler(proxy, u.URL) - docPath, timesPath := "/info/demo", "/api/v1/versions/demo.json" - if c.ecosystem == "hex" { - docPath, timesPath = "/packages/demo", "/api/packages/demo" - } - oldTimes := map[string]time.Time{cooldownOldVersion: time.Now().Add(-30 * 24 * time.Hour)} - if scenario.untimed { - oldTimes[cooldownRecentVersion] = time.Time{} - } - setDownloadMetadata(t, u, c, docPath, oldTimes) - setDownloadMetadata(t, u, c, timesPath, oldTimes) - wantStatus(t, serveGet(t, h, docPath), http.StatusOK, "") - - newTimes := map[string]time.Time{ - cooldownOldVersion: oldTimes[cooldownOldVersion], - cooldownUntimed: time.Now().Add(-time.Minute), - } - if !scenario.absent { - newTimes[cooldownRecentVersion] = time.Now().Add(-time.Hour) - } - setDownloadMetadata(t, u, c, docPath, newTimes) - expireCachedMetadata(t, proxy, c.ecosystem, docPath[1:]) - if scenario.refresh { - setDownloadMetadata(t, u, c, timesPath, newTimes) - } - u.setDown(timesPath, scenario.down) - if cached { - c.seedCachedArtifact(t, proxy, cooldownRecentVersion) - } - wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), scenario.wantStatus, "") - if fetcher.fetchCalled { - t.Error("fetched an unchecked artifact") - } - if n, _ := u.counts(timesPath); n != 2 { - t.Errorf("publish times fetched %d times, want 2", n) - } -} - -func setDownloadMetadata(t *testing.T, u *cooldownCountingUpstream, c downloadCooldownCase, path string, times map[string]time.Time) { - t.Helper() - w := httptest.NewRecorder() - c.metadata(w, httptest.NewRequest(http.MethodGet, path, nil), times) - wantStatus(t, w, http.StatusOK, "") - u.set(path, w.Body.Bytes()) -} - -func TestHexDownloadSharesPublishTimeMetadata(t *testing.T) { - u, proxy, h, _ := hexCacheSetup(t, 0) - wantStatus(t, serveGet(t, h, hexPackagePath), http.StatusOK, "") - seedPackage(t, proxy.DB, proxy.Storage.(*mockStorage), "hex", "testpkg", "1.0.0", "testpkg-1.0.0.tar", "hex bytes") - wantStatus(t, serveGet(t, h, "/tarballs/testpkg-1.0.0.tar"), http.StatusOK, "") - if n, _ := u.counts(hexAPIPath); n != 1 { - t.Fatalf("Hex API fetched %d times, want 1", n) - } -} diff --git a/internal/handler/download_cooldown_errors_test.go b/internal/handler/download_cooldown_errors_test.go deleted file mode 100644 index b214db2..0000000 --- a/internal/handler/download_cooldown_errors_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package handler - -import ( - "fmt" - "net/http" - "net/http/httptest" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - registryclient "github.com/git-pkgs/registries/client" -) - -func TestDownloadCooldownLookupFailures(t *testing.T) { - for _, c := range downloadCooldownCases() { - for _, failure := range []string{"upstream error", "invalid metadata", "rate limited"} { - for _, cached := range []bool{false, true} { - t.Run(fmt.Sprintf("%s/%s/cached=%t", c.ecosystem, failure, cached), func(t *testing.T) { - assertDownloadLookupFailure(t, c, failure, cached) - }) - } - } - } -} - -func assertDownloadLookupFailure(t *testing.T, c downloadCooldownCase, failure string, cached bool) { - t.Helper() - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if failure == "upstream error" { - w.WriteHeader(http.StatusInternalServerError) - } - _, _ = w.Write([]byte("invalid metadata")) - })) - t.Cleanup(upstream.Close) - proxy, _, _, fetcher := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "7d"} - proxy.HTTPClient = upstream.Client() - if failure == "rate limited" { - proxy.HTTPClient = &http.Client{Transport: roundTripperFunc(func(_ *http.Request) (*http.Response, error) { - return nil, ®istryclient.HeldError{Host: "upstream.example", Until: time.Now().Add(time.Minute), RetryAfter: time.Minute} - })} - } - if cached { - c.seedCachedArtifact(t, proxy, cooldownRecentVersion) - } - w := serveGet(t, c.handler(proxy, upstream.URL), c.path(cooldownRecentVersion)) - want := http.StatusBadGateway - if failure == "rate limited" { - want = http.StatusServiceUnavailable - } - if cached { - want = http.StatusOK - if w.Body.Len() == 0 { - t.Fatal("cached artifact body is empty") - } - } - wantStatus(t, w, want, "") - if fetcher.fetchCalled { - t.Fatal("lookup failure fetched an unchecked artifact") - } -} - -func TestDownloadPublishIndexesHaveShortTTLWithoutMetadataCache(t *testing.T) { - for _, c := range downloadCooldownCases() { - if c.ecosystem != "conda" && c.ecosystem != "helm" { - continue - } - t.Run(c.ecosystem, func(t *testing.T) { - var calls atomic.Int64 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) - c.metadata(w, r, map[string]time.Time{ - cooldownOldVersion: time.Now().Add(-30 * 24 * time.Hour), - cooldownRecentVersion: time.Now().Add(-time.Hour), - }) - })) - t.Cleanup(upstream.Close) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "7d"} - clock := &testClock{t: time.Now()} - proxy.now = clock.now - c.seedCachedArtifact(t, proxy, cooldownOldVersion) - c.seedCachedArtifact(t, proxy, cooldownRecentVersion) - h := c.handler(proxy, upstream.URL) - for range 3 { - wantStatus(t, serveGet(t, h, c.path(cooldownOldVersion)), http.StatusOK, "") - wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), http.StatusNotFound, "") - } - if n := calls.Load(); n != 1 { - t.Fatalf("index downloads within TTL = %d, want 1", n) - } - clock.advance(time.Minute + time.Second) - wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), http.StatusNotFound, "") - if n := calls.Load(); n != 2 { - t.Errorf("index downloads after expiry = %d, want 2", n) - } - }) - } -} diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go deleted file mode 100644 index 1fd915f..0000000 --- a/internal/handler/download_cooldown_test.go +++ /dev/null @@ -1,483 +0,0 @@ -package handler - -import ( - "compress/gzip" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// Versions every download cooldown fixture lists: one published before the -// window, one inside it, and one without a publish time. -const ( - cooldownOldVersion = "1.0.0" - cooldownRecentVersion = "2.0.0" - cooldownUntimed = "3.0.0" -) - -// downloadCooldownCase drives one ecosystem's artifact endpoint. The package -// is always named demo, so the policy key is pkg:{ecosystem}/demo. -type downloadCooldownCase struct { - ecosystem string - // cachedName is the name the artifact is cached under, and keepsTime - // whether the publish time is kept in its versions row. - cachedName string - keepsTime bool - // artifactVersion maps a release to the version the artifact is cached - // under, and filename and path to its file and download path. - artifactVersion func(version string) string - filename func(version string) string - path func(version string) string - // metadata serves the upstream document holding the publish times; a - // zero time is listed without one. - metadata func(w http.ResponseWriter, r *http.Request, times map[string]time.Time) - handler func(proxy *Proxy, upstreamURL string) http.Handler -} - -func sameVersion(version string) string { return version } - -func chartContent(version string) string { return "chart " + version } - -func downloadCooldownCases() []downloadCooldownCase { - return []downloadCooldownCase{ - { - ecosystem: "cargo", - cachedName: "demo", - keepsTime: true, - artifactVersion: sameVersion, - filename: func(v string) string { return "demo-" + v + ".crate" }, - path: func(v string) string { return "/crates/demo/" + v + "/download" }, - metadata: cargoCooldownMetadata, - handler: func(p *Proxy, u string) http.Handler { return NewCargoHandler(p, "http://proxy", u, u).Routes() }, - }, - { - ecosystem: "gem", - cachedName: "demo", - keepsTime: true, - // Platform gems carry the platform in the filename and the - // cached version, but not in the versions API number. - artifactVersion: func(v string) string { return v + "-x86_64-linux" }, - filename: func(v string) string { return "demo-" + v + "-x86_64-linux.gem" }, - path: func(v string) string { return "/gems/demo-" + v + "-x86_64-linux.gem" }, - metadata: gemCooldownMetadata, - handler: func(p *Proxy, u string) http.Handler { return NewGemHandlerWithUpstream(p, "http://proxy", u).Routes() }, - }, - { - ecosystem: "hex", - cachedName: "demo", - keepsTime: true, - artifactVersion: sameVersion, - filename: func(v string) string { return "demo-" + v + ".tar" }, - path: func(v string) string { return "/tarballs/demo-" + v + ".tar" }, - metadata: hexCooldownMetadata, - handler: func(p *Proxy, u string) http.Handler { - return NewHexHandlerWithUpstreams(p, "http://proxy", u, u).Routes() - }, - }, - { - ecosystem: "pub", - cachedName: "demo", - keepsTime: true, - artifactVersion: sameVersion, - filename: func(v string) string { return "demo-" + v + ".tar.gz" }, - path: func(v string) string { return "/packages/demo/versions/" + v + ".tar.gz" }, - metadata: pubCooldownMetadata, - handler: func(p *Proxy, u string) http.Handler { return NewPubHandlerWithUpstream(p, "http://proxy", u).Routes() }, - }, - { - // Conda caches under {channel}/{name}, but the policy is keyed by - // the name alone, as the repodata filter keys it. Each build has - // its own timestamp, so none is kept in the version's row. - ecosystem: "conda", - cachedName: "conda-forge/demo", - artifactVersion: sameVersion, - filename: func(v string) string { return "noarch/demo-" + v + "-py_0.conda" }, - path: func(v string) string { return "/conda-forge/noarch/demo-" + v + "-py_0.conda" }, - metadata: condaCooldownMetadata, - handler: func(p *Proxy, u string) http.Handler { - return NewCondaHandlerWithUpstream(p, "http://proxy", u).Routes() - }, - }, - { - // Helm caches charts under the repository and digest, so the - // chart name and time come from the index on every check. - cachedName: "stable", - ecosystem: "helm", - artifactVersion: func(v string) string { return sha256Hex(chartContent(v)) }, - filename: func(v string) string { return "demo-" + v + ".tgz" }, - path: func(v string) string { return "/stable/charts/" + sha256Hex(chartContent(v)) + "/demo-" + v + ".tgz" }, - metadata: helmCooldownMetadata, - handler: func(p *Proxy, u string) http.Handler { - return NewHelmHandler(p, "http://proxy", map[string]string{"stable": u + "/charts"}).Routes() - }, - }, - } -} - -func cargoCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { - if r.URL.Path != "/de/mo/demo" { - http.NotFound(w, r) - return - } - for version, published := range times { - line := map[string]string{"name": "demo", "vers": version} - if !published.IsZero() { - line["pubtime"] = published.Format(time.RFC3339) - } - encoded, _ := json.Marshal(line) - _, _ = fmt.Fprintf(w, "%s\n", encoded) - } -} - -func gemCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { - if r.URL.Path == "/info/demo" { - _, _ = fmt.Fprintln(w, "---") - for version := range times { - _, _ = fmt.Fprintf(w, "%s-x86_64-linux |checksum:a\n", version) - } - return - } - if r.URL.Path != "/api/v1/versions/demo.json" { - http.NotFound(w, r) - return - } - var versions []gemVersion - for version, published := range times { - v := gemVersion{Number: version, Platform: "x86_64-linux"} - if !published.IsZero() { - v.CreatedAt = published.Format(time.RFC3339) - } - versions = append(versions, v) - } - _ = json.NewEncoder(w).Encode(versions) -} - -func hexCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { - if r.URL.Path == "/packages/demo" { - var versions []string - for version := range times { - versions = append(versions, version) - } - _, _ = w.Write(buildHexSigned(buildHexPackage("demo", versions))) - return - } - if r.URL.Path != "/api/packages/demo" { - http.NotFound(w, r) - return - } - var pkg hexPackageAPI - for version, published := range times { - release := hexRelease{Version: version} - if !published.IsZero() { - release.InsertedAt = published.Format(time.RFC3339Nano) - } - pkg.Releases = append(pkg.Releases, release) - } - _ = json.NewEncoder(w).Encode(pkg) -} - -func pubCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { - if r.URL.Path != "/api/packages/demo" { - http.NotFound(w, r) - return - } - var versions []map[string]string - for version, published := range times { - v := map[string]string{"version": version} - if !published.IsZero() { - v["published"] = published.Format(time.RFC3339) - } - versions = append(versions, v) - } - _ = json.NewEncoder(w).Encode(map[string]any{"name": "demo", "versions": versions}) -} - -func condaCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { - if r.URL.Path != "/conda-forge/noarch/repodata.json" { - http.NotFound(w, r) - return - } - packages := map[string]map[string]any{} - for version, published := range times { - entry := map[string]any{"name": "demo", "version": version} - if !published.IsZero() { - entry["timestamp"] = published.UnixMilli() - } - packages["demo-"+version+"-py_0.conda"] = entry - } - // The download check asks for gzip, as proxyCached does. - w.Header().Set(headerContentEncoding, "gzip") - gz := gzip.NewWriter(w) - _ = json.NewEncoder(gz).Encode(map[string]any{"packages.conda": packages}) - _ = gz.Close() -} - -func helmCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { - if r.URL.Path != "/charts/index.yaml" { - http.NotFound(w, r) - return - } - _, _ = fmt.Fprintln(w, "apiVersion: v1\nentries:\n demo:") - for version, published := range times { - _, _ = fmt.Fprintf(w, " - digest: %s\n urls: [demo-%s.tgz]\n version: %s\n", - sha256Hex(chartContent(version)), version, version) - if !published.IsZero() { - _, _ = fmt.Fprintf(w, " created: %s\n", published.Format(time.RFC3339)) - } - } -} - -// seedCachedArtifact caches the release's artifact the way an earlier, -// cooldown-free download would have. -func (c downloadCooldownCase) seedCachedArtifact(t *testing.T, proxy *Proxy, version string) { - t.Helper() - db, store := proxy.DB, proxy.Storage.(*mockStorage) - content := "artifact " + version - if c.ecosystem == "helm" { - content = chartContent(version) - } - seedPackage(t, db, store, c.ecosystem, c.cachedName, c.artifactVersion(version), c.filename(version), content) -} - -// TestDownloadCooldownWithholdsCachedArtifacts requests each ecosystem's -// artifact endpoint for a version that was cached before the cooldown was -// raised. A version inside the window must be withheld, from the cache too; -// one past the window, or without a known publish time, is served. -func TestDownloadCooldownWithholdsCachedArtifacts(t *testing.T) { - now := time.Now().Truncate(time.Second) - times := map[string]time.Time{ - cooldownOldVersion: now.Add(-30 * 24 * time.Hour), - cooldownRecentVersion: now.Add(-time.Hour), - cooldownUntimed: {}, - } - statuses := map[string]int{ - cooldownOldVersion: http.StatusOK, - cooldownRecentVersion: http.StatusNotFound, - cooldownUntimed: http.StatusOK, - } - - for _, c := range downloadCooldownCases() { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - c.metadata(w, r, times) - })) - t.Cleanup(upstream.Close) - - for _, version := range []string{cooldownOldVersion, cooldownRecentVersion, cooldownUntimed} { - t.Run(c.ecosystem+"/"+version, func(t *testing.T) { - proxy, db, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher.fetchErr = errors.New("artifact must be served from the cache") - c.seedCachedArtifact(t, proxy, version) - proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ - canonicalPackagePURL(c.ecosystem, "demo"): "7d", - }} - - w := httptest.NewRecorder() - c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(version), nil)) - - if w.Code != statuses[version] { - t.Fatalf("status = %d, want %d (body %q)", w.Code, statuses[version], w.Body.String()) - } - if w.Code == http.StatusOK && w.Body.String() == "" { - t.Error("served an empty body") - } - if !c.keepsTime || times[version].IsZero() { - return - } - ver, err := db.GetVersionByPURL(canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(version))) - if err != nil || ver == nil || !ver.PublishedAt.Valid { - t.Fatalf("publish time not stored: %+v, %v", ver, err) - } - if got, want := ver.PublishedAt.Time, times[version]; !got.Equal(want) { - t.Errorf("stored publish time = %v, want %v", got, want) - } - }) - } - } -} - -// TestDownloadCooldownUsesStoredPublishTime checks that a stored publish -// time decides the download without asking the upstream, which here fails. -func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - })) - t.Cleanup(upstream.Close) - - for _, c := range downloadCooldownCases() { - if !c.keepsTime { - continue - } - t.Run(c.ecosystem, func(t *testing.T) { - proxy, db, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher.fetchErr = errors.New("artifact must be served from the cache") - c.seedCachedArtifact(t, proxy, cooldownRecentVersion) - versionPURL := canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(cooldownRecentVersion)) - packagePURL := canonicalPackagePURL(c.ecosystem, c.cachedName) - if err := db.SetVersionPublishedAt(versionPURL, packagePURL, time.Now().Add(-time.Hour)); err != nil { - t.Fatalf("seeding publish time: %v", err) - } - proxy.Cooldown = &cooldown.Config{Default: "7d"} - - w := httptest.NewRecorder() - c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) - if w.Code != http.StatusNotFound { - t.Fatalf("status = %d, want 404 (body %q)", w.Code, w.Body.String()) - } - }) - } -} - -// TestDownloadCooldownOtherPackagesSkipLookup checks that a download outside -// every cooldown rule is served without reading upstream metadata. Helm -// needs the index to learn the chart name, so it is left out. -func TestDownloadCooldownOtherPackagesSkipLookup(t *testing.T) { - for _, c := range downloadCooldownCases() { - if c.ecosystem == "helm" { - continue - } - t.Run(c.ecosystem, func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Errorf("unexpected upstream request %s", r.URL.Path) - http.Error(w, "unexpected", http.StatusInternalServerError) - })) - t.Cleanup(upstream.Close) - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher.fetchErr = errors.New("artifact must be served from the cache") - c.seedCachedArtifact(t, proxy, cooldownRecentVersion) - proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ - canonicalPackagePURL(c.ecosystem, "other"): "7d", - }} - - w := httptest.NewRecorder() - c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200 (body %q)", w.Code, strings.TrimSpace(w.Body.String())) - } - }) - } -} - -// TestCondaDownloadCooldownPerBuild checks that builds of one version are -// judged by their own timestamps, whichever is requested first: a rebuild -// inside the window is withheld and an old build stays available. -func TestCondaDownloadCooldownPerBuild(t *testing.T) { - now := time.Now() - repodata := map[string]any{"packages.conda": map[string]any{ - "demo-1.0.0-py_0.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-400 * 24 * time.Hour).UnixMilli()}, - "demo-1.0.0-py_1.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-time.Hour).UnixMilli()}, - }} - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(repodata) - })) - t.Cleanup(upstream.Close) - - builds := map[string]int{"py_0": http.StatusOK, "py_1": http.StatusNotFound} - for _, order := range [][]string{{"py_0", "py_1", "py_0"}, {"py_1", "py_0", "py_1"}} { - t.Run(strings.Join(order, ","), func(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher.fetchErr = errors.New("artifact must be served from the cache") - for build := range builds { - filename := "noarch/demo-1.0.0-" + build + ".conda" - seedPackage(t, db, store, "conda", "conda-forge/demo", "1.0.0", filename, "artifact "+build) - } - proxy.Cooldown = &cooldown.Config{Default: "7d"} - h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes() - - for _, build := range order { - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/noarch/demo-1.0.0-"+build+".conda", nil)) - if w.Code != builds[build] { - t.Fatalf("%s status = %d, want %d", build, w.Code, builds[build]) - } - } - }) - } -} - -// TestPublishIndexBuildsOncePerDocument checks that an index is reused for -// the same document, rebuilt for a new one, and kept briefly without a digest. -func TestPublishIndexBuildsOncePerDocument(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - builds := 0 - build := func(res metadataResult) (map[string]bool, error) { - builds++ - return map[string]bool{string(res.body): true}, nil - } - steps := []struct { - res metadataResult - want string - wantBuilds int - }{ - {metadataResult{body: []byte("a"), digest: "d1"}, "a", 1}, - {metadataResult{body: []byte("ignored"), digest: "d1"}, "a", 1}, - {metadataResult{body: []byte("b"), digest: "d2"}, "b", 2}, - {metadataResult{body: []byte("c")}, "c", 3}, - {metadataResult{body: []byte("c")}, "c", 3}, - } - for i, step := range steps { - got, err := publishIndex(proxy, "key", func() metadataResult { return step.res }, build) - if err != nil || !got[step.want] || builds != step.wantBuilds { - t.Fatalf("step %d: publishIndex = %v, %v after %d builds, want %q after %d", i, got, err, builds, step.want, step.wantBuilds) - } - } -} - -// TestCondaDownloadCooldownPastMetadataLimit checks that a cached package is -// withheld when its repodata is over metadata_max_size, as the repodata -// endpoint hides it: once only decompressed, and once already compressed. -func TestCondaDownloadCooldownPastMetadataLimit(t *testing.T) { - plain := largeRepodata(500) - compressed := gzipPayload(t, plain) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set(headerContentEncoding, "gzip") - _, _ = w.Write(compressed) - })) - t.Cleanup(upstream.Close) - - for _, tt := range []struct { - name string - limit int64 - }{ - {"decompressed over the limit", int64(len(compressed)) + 1}, - {"compressed over the limit", int64(len(compressed)) - 1}, - } { - t.Run(tt.name, func(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.MetadataMaxSize = tt.limit - fetcher.fetchErr = errors.New("artifact must be served from the cache") - for _, file := range []string{"pkg-0-old.conda", "pkg-1-new.conda"} { - seedPackage(t, db, store, "conda", "conda-forge/pkg", strings.Split(file, "-")[1], "linux-64/"+file, "artifact "+file) - } - proxy.Cooldown = &cooldown.Config{Default: "3d"} - h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes() - - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) - if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "pkg-1-new.conda") { - t.Fatalf("repodata status = %d, want 200 without pkg-1-new.conda", w.Code) - } - for file, want := range map[string]int{"pkg-0-old.conda": http.StatusOK, "pkg-1-new.conda": http.StatusNotFound} { - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+file, nil)) - if w.Code != want { - t.Errorf("%s status = %d, want %d (body %q)", file, w.Code, want, w.Body.String()) - } - } - }) - } -} diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 216a279..980e234 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -6,15 +6,13 @@ import ( "io" "net/http" "net/http/httptest" - "net/url" "strings" "testing" "time" "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/metrics" + "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" - registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" ) @@ -43,16 +41,15 @@ func seedPackageWithPURL(t *testing.T, db *database.DB, store *mockStorage, ecos t.Fatalf("failed to upsert version: %v", err) } - storagePath := ecosystem + "/" + name + "/" + version + "/" + filename + storagePath := storage.ArtifactPath(ecosystem, "", name, version, filename) store.files[storagePath] = []byte(content) - sharedArtifact := testArtifact(content, versionPURL, filename, "application/octet-stream") art := &database.Artifact{ VersionPURL: versionPURL, Filename: filename, UpstreamURL: "https://example.com/" + filename, StoragePath: sql.NullString{String: storagePath, Valid: true}, - ContentHash: sql.NullString{String: sharedArtifact.Digest.Encoded(), Valid: true}, + ContentHash: sql.NullString{String: "abc123", Valid: true}, Size: sql.NullInt64{Int64: int64(len(content)), Valid: true}, ContentType: sql.NullString{String: "application/octet-stream", Valid: true}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, @@ -206,19 +203,16 @@ func TestGemHandler_UpstreamProxy(t *testing.T) { func TestGemHandler_CacheMiss(t *testing.T) { proxy, _, _, fetcher := setupTestProxy(t) - fetchesBefore := histogramSampleCount(t, metrics.UpstreamFetchDuration.WithLabelValues("gem")) fetcher.artifact = &fetch.Artifact{ Body: io.NopCloser(strings.NewReader("fetched gem")), ContentType: "application/octet-stream", } - upstreamURL := "https://packages.example.com/gem" - h := NewGemHandlerWithUpstream(proxy, "http://localhost", upstreamURL) + h := NewGemHandler(proxy, "http://localhost") srv := httptest.NewServer(h.Routes()) defer srv.Close() - path := "/gems/sinatra-3.0.0.gem" - resp, err := http.Get(srv.URL + path) + resp, err := http.Get(srv.URL + "/gems/sinatra-3.0.0.gem") if err != nil { t.Fatalf("request failed: %v", err) } @@ -227,12 +221,6 @@ func TestGemHandler_CacheMiss(t *testing.T) { if !fetcher.fetchCalled { t.Error("expected fetcher to be called on cache miss") } - if want := upstreamURL + path; fetcher.fetchedURL != want { - t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) - } - if diff := histogramSampleCount(t, metrics.UpstreamFetchDuration.WithLabelValues("gem")) - fetchesBefore; diff != 1 { - t.Errorf("upstream fetch observations delta = %d, want 1", diff) - } } func TestGoHandler_DownloadCacheHit(t *testing.T) { @@ -331,6 +319,7 @@ func TestGoHandler_UpstreamProxy(t *testing.T) { "/golang.org/x/text/@v/v0.14.0.info", "/golang.org/x/text/@v/v0.14.0.mod", "/golang.org/x/text/@latest", + "/sumdb/sum.golang.org/lookup/golang.org/x/text@v0.14.0", } for _, path := range tests { @@ -353,13 +342,11 @@ func TestGoHandler_CacheMiss(t *testing.T) { ContentType: "application/zip", } - upstreamURL := "https://packages.example.com/go" - h := NewGoHandlerWithUpstream(proxy, "http://localhost", upstreamURL) + h := NewGoHandler(proxy, "http://localhost") srv := httptest.NewServer(h.Routes()) defer srv.Close() - path := "/example.com/mod/@v/v1.0.0.zip" - resp, err := http.Get(srv.URL + path) + resp, err := http.Get(srv.URL + "/example.com/mod/@v/v1.0.0.zip") if err != nil { t.Fatalf("request failed: %v", err) } @@ -368,9 +355,6 @@ func TestGoHandler_CacheMiss(t *testing.T) { if !fetcher.fetchCalled { t.Error("expected fetcher to be called on cache miss") } - if want := upstreamURL + path; fetcher.fetchedURL != want { - t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) - } } func TestHexHandler_DownloadCacheHit(t *testing.T) { @@ -439,13 +423,11 @@ func TestHexHandler_CacheMiss(t *testing.T) { ContentType: "application/x-tar", } - upstreamURL := "https://packages.example.com/hex" - h := NewHexHandlerWithUpstreams(proxy, "http://localhost", upstreamURL, "https://packages.example.com/hex-api") + h := NewHexHandler(proxy, "http://localhost") srv := httptest.NewServer(h.Routes()) defer srv.Close() - path := "/tarballs/plug-1.15.0.tar" - resp, err := http.Get(srv.URL + path) + resp, err := http.Get(srv.URL + "/tarballs/plug-1.15.0.tar") if err != nil { t.Fatalf("request failed: %v", err) } @@ -454,41 +436,11 @@ func TestHexHandler_CacheMiss(t *testing.T) { if !fetcher.fetchCalled { t.Error("expected fetcher to be called on cache miss") } - if want := upstreamURL + path; fetcher.fetchedURL != want { - t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) - } -} - -func TestPubHandler_CacheMiss(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("fetched pub package")), - ContentType: "application/gzip", - } - - upstreamURL := "https://packages.example.com/pub" - h := NewPubHandlerWithUpstream(proxy, "http://localhost", upstreamURL) - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - path := "/packages/flutter_bloc/versions/8.1.6.tar.gz" - resp, err := http.Get(srv.URL + path) - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if !fetcher.fetchCalled { - t.Error("expected fetcher to be called on cache miss") - } - if want := upstreamURL + path; fetcher.fetchedURL != want { - t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) - } } func TestCondaHandler_DownloadCacheHit(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) - seedPackageWithPURL(t, db, store, "conda", "main/numpy", "1.24.0", "linux-64/numpy-1.24.0-py311h64a7726_0.conda", "conda pkg") + seedPackageWithPURL(t, db, store, "conda", "main/numpy", "1.24.0", "numpy-1.24.0-py311h64a7726_0.conda", "conda pkg") h := NewCondaHandler(proxy, "http://localhost") srv := httptest.NewServer(h.Routes()) @@ -511,7 +463,7 @@ func TestCondaHandler_DownloadCacheHit(t *testing.T) { func TestCondaHandler_DownloadTarBz2CacheHit(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) - seedPackageWithPURL(t, db, store, "conda", "main/scipy", "1.11.0", "linux-64/scipy-1.11.0-py311hb2e3ea1_0.tar.bz2", "tar bz2 data") + seedPackageWithPURL(t, db, store, "conda", "main/scipy", "1.11.0", "scipy-1.11.0-py311hb2e3ea1_0.tar.bz2", "tar bz2 data") h := NewCondaHandler(proxy, "http://localhost") srv := httptest.NewServer(h.Routes()) @@ -759,8 +711,8 @@ func TestMavenHandler_MetadataProxied(t *testing.T) { paths := []string{ "/com/google/guava/guava/maven-metadata.xml", - "/com/google/guava/guava/maven-metadata.xml.sha1", - "/com/google/guava/guava/1.0-SNAPSHOT/maven-metadata.xml", + "/com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar.sha1", + "/com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar.md5", } for _, path := range paths { @@ -830,150 +782,6 @@ func TestMavenHandler_ArtifactExtensions(t *testing.T) { } } -func TestMavenHandler_ChecksumsCachedAsArtifacts(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Errorf("checksum %s proxied as metadata", r.URL.Path) - })) - defer upstream.Close() - - h := NewMavenHandler(proxy, "http://localhost", "", "") - h.upstreamURL = upstream.URL - proxy.HTTPClient = upstream.Client() - - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - for _, name := range []string{"lib-1.0.jar.sha1", "lib-1.0.pom.md5", "lib-1.0.jar.asc", "lib-1.0-bin.tar.gz.sha512"} { - path := "/com/example/lib/1.0/" + name - for i := range 2 { - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("checksum")), - ContentType: "text/plain", - } - fetcher.fetchCalled = false - - resp, err := http.Get(srv.URL + path) - if err != nil { - t.Fatalf("GET %s failed: %v", path, err) - } - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - - if resp.StatusCode != http.StatusOK || string(body) != "checksum" { - t.Errorf("GET %s: status %d body %q", path, resp.StatusCode, body) - } - if fetcher.fetchCalled != (i == 0) { - t.Errorf("GET %s request %d: fetched = %v, want %v", path, i+1, fetcher.fetchCalled, i == 0) - } - } - } -} - -// Any file named after its coordinates goes through the download path, so -// policy applies to it whatever its extension. -func TestMavenHandler_DeniedVersionBlocksEveryArtifactFile(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - setTestDenylist(t, proxy, "pkg:maven/com.google.protobuf/protoc@3.21.0") - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Errorf("denied file %s proxied to upstream", r.URL.Path) - })) - defer upstream.Close() - - h := NewMavenHandler(proxy, "http://localhost", upstream.URL, upstream.URL) - proxy.HTTPClient = upstream.Client() - - for _, name := range []string{ - "protoc-3.21.0-linux-x86_64.exe", - "protoc-3.21.0-linux-x86_64.exe.sha1", - "protoc-3.21.0.hpi", - "protoc-3.21.0.pom", - } { - fetcher.fetchCalled = false - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/com/google/protobuf/protoc/3.21.0/"+name, nil)) - if w.Code != http.StatusForbidden { - t.Errorf("GET %s: status = %d, want %d", name, w.Code, http.StatusForbidden) - } - if fetcher.fetchCalled { - t.Errorf("GET %s: fetched a denied file", name) - } - } -} - -func TestMavenHandler_UncommonArtifactFilesCached(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Errorf("artifact file %s proxied without caching", r.URL.Path) - })) - defer upstream.Close() - - h := NewMavenHandler(proxy, "http://localhost", upstream.URL, upstream.URL) - proxy.HTTPClient = upstream.Client() - - for _, path := range []string{ - "/com/google/protobuf/protoc/3.21.0/protoc-3.21.0-linux-x86_64.exe", - "/com/google/protobuf/protoc/3.21.0/protoc-3.21.0-linux-x86_64.exe.sha1", - "/org/example/plugin/1.0/plugin-1.0.hpi", - "/org/example/native/1.0/native-1.0-linux.so", - "/org/example/lib/1.0-SNAPSHOT/lib-1.0-20240101.120000-1.exe", - } { - for i := range 2 { - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("binary")), - ContentType: "application/octet-stream", - } - fetcher.fetchCalled = false - - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - - if w.Code != http.StatusOK || w.Body.String() != "binary" { - t.Errorf("GET %s: status %d body %q", path, w.Code, w.Body.String()) - } - if fetcher.fetchCalled != (i == 0) { - t.Errorf("GET %s request %d: fetched = %v, want %v", path, i+1, fetcher.fetchCalled, i == 0) - } - } - } -} - -func TestMavenHandler_NonArtifactPathsProxied(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - - var proxied []string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - proxied = append(proxied, r.URL.Path) - _, _ = fmt.Fprint(w, "listing") - })) - defer upstream.Close() - - h := NewMavenHandler(proxy, "http://localhost", upstream.URL, upstream.URL) - proxy.HTTPClient = upstream.Client() - - paths := []string{ - "/com/google/protobuf/protoc/3.21.0/", - "/com/google/protobuf/protoc/", - "/archetype-catalog.xml", - } - for _, path := range paths { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - if w.Code != http.StatusOK || w.Body.String() != "listing" { - t.Errorf("GET %s: status %d body %q", path, w.Code, w.Body.String()) - } - } - if len(proxied) != len(paths) { - t.Errorf("proxied %v, want %v", proxied, paths) - } - if fetcher.fetchCalled { - t.Error("non-artifact path went through the download path") - } -} - func TestMavenHandler_CacheMiss(t *testing.T) { proxy, _, _, fetcher := setupTestProxy(t) fetcher.artifact = &fetch.Artifact{ @@ -1074,8 +882,10 @@ func TestMavenHandler_GradlePluginMarkerFallbackAndCache(t *testing.T) { func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) { paths := map[string]string{ - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "", - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml.sha1": "sha1", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1": "sha1", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha256": "sha256", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.md5": "md5", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "", } primaryHits := map[string]int{} @@ -1133,40 +943,63 @@ func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) { } } -func TestMavenHandler_GradlePluginChecksumFallbackToPluginPortal(t *testing.T) { - primaryUpstream := "https://repo1.maven.org/maven2" - pluginPortalUpstream := "https://plugins.gradle.org/m2" - paths := []string{ - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1", - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.md5", - "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha1", - "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha256", +func TestMavenHandler_GradlePluginImplementationMetadataFallback(t *testing.T) { + paths := map[string]string{ + "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha1": "impl-sha1", + "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha256": "impl-sha256", } - for _, reqPath := range paths { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.fetchErrByURL = map[string]error{primaryUpstream + reqPath: ErrUpstreamNotFound} - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("checksum")), - ContentType: "text/plain", + primaryHits := map[string]int{} + pluginHits := map[string]int{} + + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + primaryHits[r.URL.Path]++ + if _, ok := paths[r.URL.Path]; ok { + http.NotFound(w, r) + return } + t.Fatalf("unexpected path to primary upstream: %s", r.URL.Path) + })) + defer primary.Close() - h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream) - srv := httptest.NewServer(h.Routes()) + pluginPortal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + pluginHits[r.URL.Path]++ + body, ok := paths[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + defer pluginPortal.Close() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = primary.Client() + + h := NewMavenHandler(proxy, "http://localhost", primary.URL, pluginPortal.URL) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for reqPath, wantBody := range paths { resp, err := http.Get(srv.URL + reqPath) if err != nil { t.Fatalf("GET %s failed: %v", reqPath, err) } body, _ := io.ReadAll(resp.Body) _ = resp.Body.Close() - srv.Close() - if resp.StatusCode != http.StatusOK || string(body) != "checksum" { - t.Fatalf("GET %s: status %d body %q", reqPath, resp.StatusCode, body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s: status = %d, want %d", reqPath, resp.StatusCode, http.StatusOK) } - if want := pluginPortalUpstream + reqPath; fetcher.fetchedURL != want { - t.Fatalf("GET %s: fetched URL = %q, want %q", reqPath, fetcher.fetchedURL, want) + if string(body) != wantBody { + t.Fatalf("GET %s: body = %q, want %q", reqPath, body, wantBody) + } + + if primaryHits[reqPath] == 0 { + t.Fatalf("GET %s did not hit primary upstream", reqPath) + } + if pluginHits[reqPath] == 0 { + t.Fatalf("GET %s did not hit plugin portal fallback", reqPath) } } } @@ -1282,7 +1115,7 @@ func TestConanHandler_RecipeFileCacheMiss(t *testing.T) { srv := httptest.NewServer(h.Routes()) defer srv.Close() - resp, err := http.Get(srv.URL + "/v2/conans/zlib/1.3/_/_/revisions/abc123/files/conan_export.tgz") + resp, err := http.Get(srv.URL + "/v2/files/zlib/1.3/_/_/abc123/recipe/conan_export.tgz") if err != nil { t.Fatalf("request failed: %v", err) } @@ -1292,7 +1125,7 @@ func TestConanHandler_RecipeFileCacheMiss(t *testing.T) { t.Error("expected fetcher to be called on cache miss") } - want := "https://center.conan.io/v2/conans/zlib/1.3/_/_/revisions/abc123/files/conan_export.tgz" + want := "https://center.conan.io/v2/files/zlib/1.3/_/_/abc123/recipe/conan_export.tgz" if fetcher.fetchedURL != want { t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) } @@ -1309,7 +1142,7 @@ func TestConanHandler_PackageFileCacheMiss(t *testing.T) { srv := httptest.NewServer(h.Routes()) defer srv.Close() - resp, err := http.Get(srv.URL + "/v2/conans/zlib/1.3/_/_/revisions/abc123/packages/def456/revisions/ghi789/files/conan_package.tgz") + resp, err := http.Get(srv.URL + "/v2/files/zlib/1.3/_/_/abc123/package/def456/ghi789/conan_package.tgz") if err != nil { t.Fatalf("request failed: %v", err) } @@ -1319,7 +1152,7 @@ func TestConanHandler_PackageFileCacheMiss(t *testing.T) { t.Error("expected fetcher to be called on cache miss") } - want := "https://center.conan.io/v2/conans/zlib/1.3/_/_/revisions/abc123/packages/def456/revisions/ghi789/files/conan_package.tgz" + want := "https://center.conan.io/v2/files/zlib/1.3/_/_/abc123/package/def456/ghi789/conan_package.tgz" if fetcher.fetchedURL != want { t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) } @@ -1332,7 +1165,7 @@ func TestDebianHandler_DownloadCacheMiss(t *testing.T) { ContentType: "application/vnd.debian.binary-package", } - h := NewDebianHandler(proxy, "http://localhost", "", nil) + h := NewDebianHandler(proxy, "http://localhost") srv := httptest.NewServer(h.Routes()) defer srv.Close() @@ -1378,134 +1211,3 @@ func TestRPMHandler_DownloadCacheMiss(t *testing.T) { t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) } } - -func TestMavenHandler_UpstreamHeldReturns503(t *testing.T) { - held := ®istryclient.HeldError{Host: "repo1.maven.org", Until: time.Now().Add(90 * time.Second), RetryAfter: 90 * time.Second} - - t.Run("artifact", func(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.fetchErr = fmt.Errorf("fetching artifact: %w", &url.Error{Op: "Get", URL: "https://repo1.maven.org/x", Err: held}) - - h := NewMavenHandler(proxy, "http://localhost", "", "") - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/com/example/lib/1.0/lib-1.0.jar") - if err != nil { - t.Fatalf("request failed: %v", err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") != "90" { - t.Errorf("status %d Retry-After %q, want 503 with Retry-After 90", resp.StatusCode, resp.Header.Get("Retry-After")) - } - }) - - t.Run("metadata", func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = &http.Client{Transport: roundTripperFunc(func(*http.Request) (*http.Response, error) { - return nil, held - })} - - h := NewMavenHandler(proxy, "http://localhost", "", "") - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/com/example/lib/maven-metadata.xml") - if err != nil { - t.Fatalf("request failed: %v", err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") != "90" { - t.Errorf("status %d Retry-After %q, want 503 with Retry-After 90", resp.StatusCode, resp.Header.Get("Retry-After")) - } - }) -} - -type roundTripperFunc func(*http.Request) (*http.Response, error) - -func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { return fn(req) } - -func TestMavenHandler_HeldArtifactMissesSkipCircuitBreaker(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - upstreamCalls := 0 - gate := registryclient.NewRateLimitGate(roundTripperFunc(func(req *http.Request) (*http.Response, error) { - upstreamCalls++ - return &http.Response{ - StatusCode: http.StatusTooManyRequests, - Header: http.Header{"Retry-After": []string{"600"}}, - Body: http.NoBody, - Request: req, - }, nil - })) - req, _ := http.NewRequest(http.MethodGet, "https://repo1.maven.org/maven2/x", nil) - if resp, err := gate.RoundTrip(req); err == nil { - _ = resp.Body.Close() - } - breaker := fetch.NewCircuitBreakerFetcher(fetch.NewFetcher(fetch.WithHTTPClient(&http.Client{Transport: gate}))) - proxy.Fetcher = breaker - proxy.RateLimits = gate - - h := NewMavenHandler(proxy, "http://localhost", "", "") - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - for i := range 8 { - resp, err := http.Get(fmt.Sprintf("%s/com/example/lib/1.%d/lib-1.%d.jar", srv.URL, i, i)) - if err != nil { - t.Fatalf("request %d failed: %v", i, err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") == "" { - t.Fatalf("request %d: status %d Retry-After %q, want 503 with Retry-After", i, resp.StatusCode, resp.Header.Get("Retry-After")) - } - } - if upstreamCalls != 1 { - t.Errorf("upstream calls = %d, want only the one that started the hold", upstreamCalls) - } - if state := breaker.GetBreakerState()["repo1.maven.org"]; state == "open" { - t.Errorf("breaker for repo1.maven.org is open after held misses, want closed") - } -} - -func TestHandlers_UpstreamHeldMetadataReturns503(t *testing.T) { - held := ®istryclient.HeldError{Host: "upstream.example", Until: time.Now().Add(42 * time.Second), RetryAfter: 42 * time.Second} - tests := []struct { - name string - handler func(*Proxy) http.Handler - path string - }{ - {"npm metadata", func(p *Proxy) http.Handler { return NewNPMHandler(p, "http://localhost", "").Routes() }, "/lodash"}, - {"pypi simple", func(p *Proxy) http.Handler { - return NewPyPIHandlerWithUpstreams(p, "http://localhost", "", "").Routes() - }, "/simple/requests/"}, - {"pypi json", func(p *Proxy) http.Handler { - return NewPyPIHandlerWithUpstreams(p, "http://localhost", "", "").Routes() - }, "/pypi/requests/json"}, - {"cargo index", func(p *Proxy) http.Handler { - return NewCargoHandler(p, "http://localhost", "", "").Routes() - }, "/se/rd/serde"}, - {"pub metadata", func(p *Proxy) http.Handler { return NewPubHandler(p, "http://localhost").Routes() }, "/api/packages/http"}, - {"nuget service index", func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://localhost").Routes() }, "/v3/index.json"}, - {"nuget search", func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://localhost").Routes() }, "/query?q=json"}, - {"conda repodata", func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://localhost").Routes() }, "/conda-forge/noarch/repodata.json"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = &http.Client{Transport: roundTripperFunc(func(*http.Request) (*http.Response, error) { - return nil, held - })} - srv := httptest.NewServer(tt.handler(proxy)) - defer srv.Close() - - resp, err := http.Get(srv.URL + tt.path) - if err != nil { - t.Fatalf("request failed: %v", err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") != "42" { - t.Errorf("status %d Retry-After %q, want 503 with Retry-After 42", resp.StatusCode, resp.Header.Get("Retry-After")) - } - }) - } -} diff --git a/internal/handler/fetch_lease.go b/internal/handler/fetch_lease.go deleted file mode 100644 index e49b7ef..0000000 --- a/internal/handler/fetch_lease.go +++ /dev/null @@ -1,183 +0,0 @@ -package handler - -import ( - "context" - "crypto/rand" - "crypto/sha256" - "encoding/hex" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/metrics" -) - -// Instances sharing a Postgres database coalesce cache misses through fetch -// leases, as coalesceFetch and coalesceMetadata do within one instance. The -// instance that runs the fetch holds the lease for the miss and renews it -// while it downloads. Another instance missing the same key waits for the -// stored result and serves that, or takes the lease over if the holder -// releases it without storing anything or stops renewing it. -var ( - fetchLeaseTTL = 30 * time.Second - fetchLeaseRenew = 10 * time.Second - // The wait for another instance's fetch polls the cache, starting - // quickly for small files and backing off for large ones. - fetchLeasePollMin = 50 * time.Millisecond - fetchLeasePollMax = 2 * time.Second -) - -// fetchLeaseKey turns a coalescing key, which can hold long URLs and NUL -// separators, into a fixed-size table key. -func fetchLeaseKey(kind, key string) string { - sum := sha256.Sum256([]byte(kind + "\x00" + key)) - return hex.EncodeToString(sum[:]) -} - -func newLeaseOwner() string { - var b [16]byte - _, _ = rand.Read(b[:]) - return hex.EncodeToString(b[:]) -} - -// fetchLeasesEnabled reports whether misses are coalesced across instances. -func (p *Proxy) fetchLeasesEnabled() bool { - return p.DB != nil && p.DB.Dialect() == database.DialectPostgres -} - -// leaseOutcome says how a caller goes on after awaitFetchLease. -type leaseOutcome int - -const ( - // leaseFetch: this instance fetches, and calls release afterwards. - leaseFetch leaseOutcome = iota - // leaseStored: ready found the result another instance stored. - leaseStored - // leaseFailed: the fetch this caller waited on failed. It fetches - // without the lease, as every instance that waited does at the same - // time, or serves a stale copy, rather than queueing behind the others. - leaseFailed -) - -// awaitFetchLease takes the lease on key before this instance fetches it from -// upstream. While another instance holds the lease, it polls ready, which -// reports whether that instance's result is now stored and, if so, keeps it -// for the caller. release is to be called with whether the fetch failed once -// it is done; err is set when ctx ended while waiting. -// -// A failing lease query is logged and the fetch goes ahead without a lease, -// so a database hiccup costs a duplicate fetch rather than the request. -func (p *Proxy) awaitFetchLease(ctx context.Context, kind, key string, ready func() bool) (release func(failed bool), outcome leaseOutcome, err error) { - noop := func(bool) {} - if !p.fetchLeasesEnabled() { - return noop, leaseFetch, nil - } - leaseKey := fetchLeaseKey(kind, key) - owner := newLeaseOwner() - delay := fetchLeasePollMin - waited := false - for { - acquired, err := p.DB.AcquireFetchLease(leaseKey, owner, fetchLeaseTTL) - if err != nil { - p.Logger.Warn("fetch lease unavailable, fetching without it", "kind", kind, "error", err) - return noop, leaseFetch, nil - } - if acquired { - // The holder may have stored the result and released the lease - // between this caller's last poll and taking it. - if waited && ready() { - p.releaseFetchLease(leaseKey, owner) - return noop, leaseStored, nil - } - return p.holdFetchLease(leaseKey, owner), leaseFetch, nil - } - if !waited { - metrics.RecordFetchLeaseWait(kind) - waited = true - } - outcome, done, err := p.waitForLeaseHolder(ctx, kind, leaseKey, ready, &delay) - if done || err != nil { - return noop, outcome, err - } - } -} - -// waitForLeaseHolder polls ready while another instance holds the lease on -// leaseKey. It is done when ready found the result, when the fetch it waited -// on failed, or when the lease can't be read; otherwise it returns once the -// lease is free to take. -func (p *Proxy) waitForLeaseHolder(ctx context.Context, kind, leaseKey string, ready func() bool, delay *time.Duration) (outcome leaseOutcome, done bool, err error) { - holder := "" - for { - current, failed, held, err := p.DB.FetchLeaseHolder(leaseKey) - if err != nil { - p.Logger.Warn("fetch lease unavailable, fetching without it", "kind", kind, "error", err) - return leaseFetch, true, nil - } - if failed && current == holder { - return leaseFailed, true, nil - } - if !held || failed { - // Released, expired, or failed before this caller waited on - // it: free to take. - return leaseFetch, false, nil - } - holder = current - timer := time.NewTimer(*delay) - select { - case <-ctx.Done(): - timer.Stop() - return leaseFetch, true, ctx.Err() - case <-timer.C: - } - if ready() { - return leaseStored, true, nil - } - *delay = min(*delay+*delay, fetchLeasePollMax) - } -} - -// holdFetchLease renews owner's lease on leaseKey until the returned release -// is called. release drops the lease, or marks it failed so instances -// waiting on it stop waiting. -func (p *Proxy) holdFetchLease(leaseKey, owner string) func(failed bool) { - stop := make(chan struct{}) - done := make(chan struct{}) - go func() { - defer close(done) - ticker := time.NewTicker(fetchLeaseRenew) - defer ticker.Stop() - for { - select { - case <-stop: - return - case <-ticker.C: - // A lease that lapsed, say after a long pause, lets another - // instance fetch too. That costs a duplicate fetch, nothing - // else, so the fetch here carries on either way. - if held, err := p.DB.RenewFetchLease(leaseKey, owner, fetchLeaseTTL); err != nil || !held { - p.Logger.Debug("fetch lease lapsed", "error", err) - } - } - } - }() - return func(failed bool) { - close(stop) - <-done - if !failed { - p.releaseFetchLease(leaseKey, owner) - return - } - if err := p.DB.FailFetchLease(leaseKey, owner); err != nil { - // Waiting instances take the lease once it expires. - p.Logger.Warn("failed to mark fetch lease failed", "error", err) - } - } -} - -func (p *Proxy) releaseFetchLease(leaseKey, owner string) { - if err := p.DB.ReleaseFetchLease(leaseKey, owner); err != nil { - // The lease expires on its own; until then other instances wait - // on it and then fetch. - p.Logger.Warn("failed to release fetch lease", "error", err) - } -} diff --git a/internal/handler/fetch_lease_test.go b/internal/handler/fetch_lease_test.go deleted file mode 100644 index 0be8d42..0000000 --- a/internal/handler/fetch_lease_test.go +++ /dev/null @@ -1,428 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/metrics" - "github.com/git-pkgs/registries/fetch" - "github.com/prometheus/client_golang/prometheus/testutil" -) - -// gateFetcher counts upstream fetches, signals entered when the first one -// starts and holds every fetch until release is closed. -type gateFetcher struct { - calls atomic.Int64 - content string - err error - entered chan struct{} - release chan struct{} - once sync.Once - opened sync.Once -} - -// newGateFetcher returns a gateFetcher that is released when the test ends -// at the latest, so a failing test does not leave fetches hanging. -func newGateFetcher(t *testing.T, content string, err error) *gateFetcher { - f := &gateFetcher{content: content, err: err, entered: make(chan struct{}), release: make(chan struct{})} - t.Cleanup(f.open) - return f -} - -func (f *gateFetcher) open() { f.opened.Do(func() { close(f.release) }) } - -func (f *gateFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { - return f.FetchWithHeaders(ctx, url, nil) -} - -func (f *gateFetcher) FetchWithHeaders(ctx context.Context, _ string, _ http.Header) (*fetch.Artifact, error) { - f.calls.Add(1) - f.once.Do(func() { close(f.entered) }) - select { - case <-f.release: - case <-ctx.Done(): - return nil, ctx.Err() - } - if f.err != nil { - return nil, f.err - } - return &fetch.Artifact{Body: io.NopCloser(strings.NewReader(f.content)), ContentType: "application/gzip"}, nil -} - -func (f *gateFetcher) Head(context.Context, string) (int64, string, error) { return 0, "", nil } - -// sharedInstances returns two proxies sharing one Postgres database and one -// storage, as two replicas would. It skips without PROXY_DATABASE_URL. -func sharedInstances(t *testing.T) (*Proxy, *Proxy, *database.DB) { - t.Helper() - db := openTestPostgres(t) - if db == nil { - t.Skip("PROXY_DATABASE_URL not set") - } - store := newMockStorage() - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - a := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), logger) - b := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), logger) - return a, b, db -} - -const leaseTestURL = "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz" - -func fetchLeftPad(p *Proxy) (string, error) { - return fetchLeftPadCtx(context.Background(), p) -} - -func fetchLeftPadCtx(ctx context.Context, p *Proxy) (string, error) { - res, err := p.GetOrFetchArtifactFromURL(ctx, "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", leaseTestURL) - if err != nil { - return "", err - } - defer func() { _ = res.Reader.Close() }() - b, err := io.ReadAll(res.Reader) - return string(b), err -} - -type fetchResult struct { - body string - err error -} - -func goFetchLeftPad(ctx context.Context, p *Proxy) chan fetchResult { - ch := make(chan fetchResult, 1) - go func() { - body, err := fetchLeftPadCtx(ctx, p) - ch <- fetchResult{body, err} - }() - return ch -} - -// waitForLeaseWaits returns once the lease wait counter for kind has passed -// from. -func waitForLeaseWaits(t *testing.T, kind string, from float64) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(kind)) <= from { - if time.Now().After(deadline) { - t.Fatal("timed out waiting for the second instance to wait on the lease") - } - time.Sleep(time.Millisecond) - } -} - -// waitForArtifactWaiters returns once n callers have joined p's fetch of -// left-pad. -func waitForArtifactWaiters(t *testing.T, p *Proxy, n int) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for { - p.fetchMu.Lock() - joined := 0 - for _, f := range p.inFlight { - joined += f.waiters - } - p.fetchMu.Unlock() - if joined >= n { - return - } - if time.Now().After(deadline) { - t.Fatal("timed out waiting for a caller to join the fetch") - } - time.Sleep(time.Millisecond) - } -} - -func TestFetchLeaseCoalescesArtifactAcrossInstances(t *testing.T) { - a, b, _ := sharedInstances(t) - const content = "left-pad tarball" - fa := newGateFetcher(t, content, nil) - fb := newGateFetcher(t, content, nil) - fb.open() - a.Fetcher, b.Fetcher = fa, fb - - waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) - resA := goFetchLeftPad(context.Background(), a) - <-fa.entered - resB := goFetchLeftPad(context.Background(), b) - waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) - fa.open() - - for name, ch := range map[string]chan fetchResult{"a": resA, "b": resB} { - r := <-ch - if r.err != nil || r.body != content { - t.Errorf("instance %s: body %q, err %v", name, r.body, r.err) - } - } - if got := fa.calls.Load() + fb.calls.Load(); got != 1 { - t.Errorf("upstream fetches = %d (a %d, b %d), want 1", got, fa.calls.Load(), fb.calls.Load()) - } -} - -// TestFetchLeaseFailedFetchLetsOtherInstanceFetch checks that an instance -// waiting on a fetch that fails takes the lease over and fetches itself, -// rather than waiting until the lease expires. -func TestFetchLeaseFailedFetchLetsOtherInstanceFetch(t *testing.T) { - a, b, _ := sharedInstances(t) - const content = "left-pad tarball" - fa := newGateFetcher(t, "", errors.New("upstream down")) - fb := newGateFetcher(t, content, nil) - fb.open() - a.Fetcher, b.Fetcher = fa, fb - - waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) - resA := goFetchLeftPad(context.Background(), a) - <-fa.entered - resB := goFetchLeftPad(context.Background(), b) - waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) - started := time.Now() - fa.open() - - if r := <-resA; r.err == nil { - t.Error("instance a: want the upstream error") - } - r := <-resB - if r.err != nil || r.body != content { - t.Fatalf("instance b: body %q, err %v", r.body, r.err) - } - if fb.calls.Load() != 1 { - t.Errorf("b's upstream fetches = %d, want 1", fb.calls.Load()) - } - if waited := time.Since(started); waited >= fetchLeaseTTL { - t.Errorf("b waited %s, as long as the lease lasts", waited) - } -} - -// TestFetchLeaseExpiredIsTakenOver checks that a lease left by an instance -// that stopped while fetching holds the others up only until it expires. -func TestFetchLeaseExpiredIsTakenOver(t *testing.T) { - _, b, db := sharedInstances(t) - const content = "left-pad tarball" - fb := newGateFetcher(t, content, nil) - fb.open() - b.Fetcher = fb - - key := artifactCoalesceKey("pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz", leaseTestURL, "", "") - if ok, err := db.AcquireFetchLease(fetchLeaseKey(metrics.CacheKindArtifact, key), "stopped-instance", 300*time.Millisecond); err != nil || !ok { - t.Fatalf("taking the lease for the stopped instance: %v, %v", ok, err) - } - started := time.Now() - body, err := fetchLeftPad(b) - if err != nil || body != content { - t.Fatalf("body %q, err %v", body, err) - } - if waited := time.Since(started); waited < 250*time.Millisecond { - t.Errorf("fetched after %s, before the lease expired; is the test key right?", waited) - } -} - -func TestFetchLeaseCoalescesMetadataAcrossInstances(t *testing.T) { - a, b, _ := sharedInstances(t) - u := newMetadataUpstream(t, http.StatusOK) - var released sync.Once - release := func() { released.Do(func() { close(u.release) }) } - t.Cleanup(release) - for _, p := range []*Proxy{a, b} { - p.HTTPClient = u.Client() - p.CacheMetadata = true - p.MetadataTTL = time.Minute - } - - waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindMetadata)) - resA := startMetadataCalls(a, context.Background(), 1, u.URL+"/left-pad", "application/json")[0] - <-u.entered - resB := startMetadataCalls(b, context.Background(), 1, u.URL+"/left-pad", "application/json")[0] - waitForLeaseWaits(t, metrics.CacheKindMetadata, waits) - release() - - for name, ch := range map[string]chan metadataCall{"a": resA, "b": resB} { - r := <-ch - if r.err != nil || !strings.Contains(string(r.body), "application/json") { - t.Errorf("instance %s: body %q, err %v", name, r.body, r.err) - } - } - if got := u.calls.Load(); got != 1 { - t.Errorf("upstream metadata requests = %d, want 1", got) - } -} - -// TestFetchLeaseNotUsedWithoutMetadataCache checks that metadata misses are -// not held up when there is no stored copy to wait for. -func TestFetchLeaseNotUsedWithoutMetadataCache(t *testing.T) { - a, b, _ := sharedInstances(t) - u := newMetadataUpstream(t, http.StatusOK) - a.HTTPClient, b.HTTPClient = u.Client(), u.Client() - - resA := startMetadataCalls(a, context.Background(), 1, u.URL+"/left-pad", "application/json")[0] - <-u.entered - resB := startMetadataCalls(b, context.Background(), 1, u.URL+"/left-pad", "application/json")[0] - deadline := time.Now().Add(5 * time.Second) - for u.calls.Load() < 2 { - if time.Now().After(deadline) { - t.Fatal("b did not fetch while a was fetching") - } - time.Sleep(time.Millisecond) - } - close(u.release) - for _, ch := range []chan metadataCall{resA, resB} { - if r := <-ch; r.err != nil { - t.Errorf("unexpected error: %v", r.err) - } - } -} - -// TestFetchLeaseWaitOutlivesDetachedCaller checks that a caller waiting on -// another instance's fetch keeps waiting after its client goes away when its -// fetch is detached, so callers that joined it still get the artifact. -func TestFetchLeaseWaitOutlivesDetachedCaller(t *testing.T) { - a, b, _ := sharedInstances(t) - const content = "left-pad tarball" - fa := newGateFetcher(t, content, nil) - fb := newGateFetcher(t, content, nil) - fb.open() - a.Fetcher, b.Fetcher = fa, fb - - waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) - resA := goFetchLeftPad(context.Background(), a) - <-fa.entered - leadCtx, cancelLead := context.WithCancel(DetachUpstreamFetch(context.Background())) - defer cancelLead() - resLead := goFetchLeftPad(leadCtx, b) - waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) - resJoined := goFetchLeftPad(context.Background(), b) - waitForArtifactWaiters(t, b, 1) - cancelLead() - fa.open() - <-resLead - - for name, ch := range map[string]chan fetchResult{"a": resA, "b's joined caller": resJoined} { - r := <-ch - if r.err != nil || r.body != content { - t.Errorf("%s: body %q, err %v", name, r.body, r.err) - } - } - if got := fb.calls.Load(); got != 0 { - t.Errorf("b fetched %d times, want 0", got) - } -} - -// TestFetchLeaseFailedFetchWaitersFetchInParallel checks that once a fetch -// fails, the instances waiting on it fetch at the same time rather than one -// after another, so a hanging upstream costs each one timeout. -func TestFetchLeaseFailedFetchWaitersFetchInParallel(t *testing.T) { - a, b, db := sharedInstances(t) - c := NewProxy(db, b.Storage, &mockFetcher{}, fetch.NewResolver(), b.Logger) - const content = "left-pad tarball" - fa := newGateFetcher(t, "", errors.New("upstream down")) - fb := newGateFetcher(t, content, nil) - fc := newGateFetcher(t, content, nil) - a.Fetcher, b.Fetcher, c.Fetcher = fa, fb, fc - - waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) - resA := goFetchLeftPad(context.Background(), a) - <-fa.entered - resB := goFetchLeftPad(context.Background(), b) - waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) - resC := goFetchLeftPad(context.Background(), c) - waitForLeaseWaits(t, metrics.CacheKindArtifact, waits+1) - fa.open() - if r := <-resA; r.err == nil { - t.Fatal("instance a: want the upstream error") - } - - for name, entered := range map[string]chan struct{}{"b": fb.entered, "c": fc.entered} { - select { - case <-entered: - case <-time.After(5 * time.Second): - t.Fatalf("%s did not fetch while the other waiting instance was fetching", name) - } - } - fb.open() - fc.open() - for name, ch := range map[string]chan fetchResult{"b": resB, "c": resC} { - if r := <-ch; r.err != nil || r.body != content { - t.Errorf("instance %s: body %q, err %v", name, r.body, r.err) - } - } -} - -// TestFetchLeaseFailedMetadataServesStale checks that an instance waiting on -// a metadata fetch that fails serves its stale copy at once, as the holder -// does, instead of asking the failing upstream again. -func TestFetchLeaseFailedMetadataServesStale(t *testing.T) { - a, b, _ := sharedInstances(t) - var status atomic.Int32 - status.Store(http.StatusOK) - var calls atomic.Int32 - entered := make(chan struct{}, 1) - release := make(chan struct{}) - var released sync.Once - open := func() { released.Do(func() { close(release) }) } - t.Cleanup(open) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) - if code := int(status.Load()); code != http.StatusOK { - entered <- struct{}{} - <-release - w.WriteHeader(code) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"name":"left-pad"}`)) - })) - t.Cleanup(upstream.Close) - for _, p := range []*Proxy{a, b} { - p.HTTPClient = upstream.Client() - p.CacheMetadata = true - p.MetadataTTL = 50 * time.Millisecond - } - fetchMeta := func(p *Proxy) chan metadataCall { - ch := make(chan metadataCall, 1) - go func() { - body, _, err := p.FetchOrCacheMetadata(context.Background(), "npm", "left-pad", upstream.URL+"/left-pad", "application/json") - ch <- metadataCall{body, err} - }() - return ch - } - if r := <-fetchMeta(a); r.err != nil { - t.Fatalf("seeding the cache: %v", r.err) - } - time.Sleep(100 * time.Millisecond) - - status.Store(http.StatusServiceUnavailable) - waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindMetadata)) - resA := fetchMeta(a) - <-entered - resB := fetchMeta(b) - waitForLeaseWaits(t, metrics.CacheKindMetadata, waits) - open() - - for name, ch := range map[string]chan metadataCall{"a": resA, "b": resB} { - if r := <-ch; r.err != nil || string(r.body) != `{"name":"left-pad"}` { - t.Errorf("instance %s: body %q, err %v; want the stale copy", name, r.body, r.err) - } - } - if got := calls.Load(); got != 2 { - t.Errorf("upstream requests = %d, want 2: the seed and a's failed refresh", got) - } -} - -func TestFetchLeaseNotUsedOnSQLite(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - release, outcome, err := proxy.awaitFetchLease(context.Background(), metrics.CacheKindArtifact, "k", func() bool { - t.Error("ready polled on SQLite") - return false - }) - if err != nil || outcome != leaseFetch { - t.Fatalf("awaitFetchLease = %v, %v; want to fetch at once", outcome, err) - } - release(false) -} diff --git a/internal/handler/fetch_path_test.go b/internal/handler/fetch_path_test.go deleted file mode 100644 index 4f7602a..0000000 --- a/internal/handler/fetch_path_test.go +++ /dev/null @@ -1,163 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "net/http" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/registries/fetch" -) - -// These tests cover fetches of one artifact that do not share a coalescing -// key because their URLs differ. Each must keep its own stored object. - -const ( - fetchPathURLA = "https://mirror-a.example/pkg-1.0.0.tgz" - fetchPathURLB = "https://mirror-b.example/pkg-1.0.0.tgz" -) - -// barrierFetcher serves a body per URL and holds each fetch until both have -// started, so both requests are past the cache check before either commits. -type barrierFetcher struct { - bodies map[string]string - arrived sync.WaitGroup -} - -func newBarrierFetcher(bodies map[string]string) *barrierFetcher { - f := &barrierFetcher{bodies: bodies} - f.arrived.Add(len(bodies)) - return f -} - -func (f *barrierFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { - return f.FetchWithHeaders(ctx, url, nil) -} - -func (f *barrierFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) { - f.arrived.Done() - f.arrived.Wait() - return artifactBody(f.bodies[url]), nil -} - -func (f *barrierFetcher) Head(_ context.Context, _ string) (int64, string, error) { - return 0, "", nil -} - -// gatedStorage holds the first Open until ready reports true, to order it -// after the other request's store or delete. -type gatedStorage struct { - *mockStorage - ready func(stores, deletes int32) bool - stores atomic.Int32 - deletes atomic.Int32 - opened atomic.Bool - release chan struct{} - releaseMu sync.Once -} - -func newGatedStorage(store *mockStorage, ready func(stores, deletes int32) bool) *gatedStorage { - return &gatedStorage{mockStorage: store, ready: ready, release: make(chan struct{})} -} - -func (g *gatedStorage) check() { - if g.ready(g.stores.Load(), g.deletes.Load()) { - g.releaseMu.Do(func() { close(g.release) }) - } -} - -func (g *gatedStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { - size, hash, err := g.mockStorage.Store(ctx, path, r) - g.stores.Add(1) - g.check() - return size, hash, err -} - -func (g *gatedStorage) Delete(ctx context.Context, path string) error { - err := g.mockStorage.Delete(ctx, path) - g.deletes.Add(1) - g.check() - return err -} - -func (g *gatedStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) { - if g.opened.CompareAndSwap(false, true) { - select { - case <-g.release: - case <-ctx.Done(): - return nil, ctx.Err() - } - } - return g.mockStorage.Open(ctx, path) -} - -func readResult(res *CacheResult) string { - b, _ := io.ReadAll(res.Reader) - _ = res.Reader.Close() - return string(b) -} - -// TestFetchesWithDifferentURLsServeTheirOwnBytes has both fetches store before -// either opens. Sharing one object, one of them would serve the other's bytes. -func TestFetchesWithDifferentURLsServeTheirOwnBytes(t *testing.T) { - proxy, _, store, _ := setupTestProxy(t) - bodies := map[string]string{fetchPathURLA: "bytes from a", fetchPathURLB: "bytes from b"} - proxy.Fetcher = newBarrierFetcher(bodies) - proxy.Storage = newGatedStorage(store, func(stores, _ int32) bool { return stores == 2 }) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - var wg sync.WaitGroup - for url, want := range bodies { - wg.Go(func() { - res, err := proxy.GetOrFetchArtifactFromURL(ctx, "npm", "pkg", "1.0.0", staleFilename, url) - if err != nil { - t.Errorf("fetch of %s: %v", url, err) - return - } - if got := readResult(res); got != want { - t.Errorf("fetch of %s served %q, want %q", url, got, want) - } - }) - } - wg.Wait() -} - -// TestDigestMismatchLeavesAnotherFetchsObject has one fetch discard bytes that -// fail the declared digest before another fetch, which stored good bytes, -// opens them. Sharing one object, the discard would delete the good bytes. -func TestDigestMismatchLeavesAnotherFetchsObject(t *testing.T) { - proxy, _, store, _ := setupTestProxy(t) - proxy.Fetcher = newBarrierFetcher(map[string]string{fetchPathURLA: "good bytes", fetchPathURLB: "tampered bytes"}) - proxy.Storage = newGatedStorage(store, func(_, deletes int32) bool { return deletes == 1 }) - declared := "sha256:" + sha256Hex("good bytes") - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - var wg sync.WaitGroup - wg.Go(func() { - res, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "npm", "pkg", "1.0.0", staleFilename, fetchPathURLA, declared) - if err != nil { - t.Errorf("good fetch: %v", err) - return - } - if got := readResult(res); got != "good bytes" { - t.Errorf("good fetch served %q", got) - } - }) - wg.Go(func() { - _, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "npm", "pkg", "1.0.0", staleFilename, fetchPathURLB, declared) - if !errors.Is(err, ErrArtifactDigestMismatch) { - t.Errorf("tampered fetch: got %v, want a digest mismatch", err) - } - }) - wg.Wait() - - if _, ok := store.files[recordedStoragePath(t, proxy.DB, staleVersionPURL, staleFilename)]; !ok { - t.Error("the recorded object was deleted") - } -} diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go deleted file mode 100644 index c37d07d..0000000 --- a/internal/handler/filename_download.go +++ /dev/null @@ -1,54 +0,0 @@ -package handler - -import ( - "net/http" - "strings" - "time" -) - -type filenameDownload struct { - ecosystem string - upstreamURL string - suffix string - parseErr string - fetchErr string - parse func(string) (name, version string) - // publishTime reads a version's publish time from upstream metadata for - // the download-time cooldown check. - publishTime func(r *http.Request, name, version string) (time.Time, bool, bool, error) -} - -func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d filenameDownload) { - filename := r.PathValue("filename") - if filename == "" || !strings.HasSuffix(filename, d.suffix) { - http.Error(w, "invalid filename", http.StatusBadRequest) - return - } - - name, version := d.parse(filename) - if name == "" || version == "" { - http.Error(w, d.parseErr, http.StatusBadRequest) - return - } - - p.Logger.Debug(d.ecosystem+" download request", - "name", name, "version", version, "filename", filename) - - if p.withholdInCooldown(w, r, downloadVersion{ - ecosystem: d.ecosystem, policyName: name, storedName: name, version: version, cacheName: name, filename: filename, - lookup: func() (time.Time, bool, bool, error) { return d.publishTime(r, name, version) }, - }) { - return - } - - downloadURL := d.upstreamURL + r.URL.Path - result, err := p.GetOrFetchArtifactFromURL( - r.Context(), d.ecosystem, name, version, filename, downloadURL, - ) - if err != nil { - p.serveArtifactError(w, err, d.fetchErr) - return - } - - ServeArtifactRequest(w, r, result) -} diff --git a/internal/handler/freebsd.go b/internal/handler/freebsd.go deleted file mode 100644 index aceffa4..0000000 --- a/internal/handler/freebsd.go +++ /dev/null @@ -1,256 +0,0 @@ -package handler - -import ( - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "net/http" - "regexp" - "strings" - - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/purl" -) - -const ( - freebsdEcosystem = "freebsd" - // defaultFreeBSDRepositoryName is the repository name used when no - // upstream.freebsd repositories are configured. - defaultFreeBSDRepositoryName = "freebsd" - // defaultFreeBSDUpstream is the official FreeBSD package mirror. - defaultFreeBSDUpstream = "https://pkg.freebsd.org" - freebsdNamespace = "freebsd" - freebsdMatchCount = 3 // full match + name + version - freebsdHashedDir = "Hashed" - freebsdPackageMedia = "application/octet-stream" -) - -// Packages are stored as pkg:bsd/freebsd/{name}, whichever configured -// repository they came from. -func init() { - retention.Default.Register(retention.Spec{Key: freebsdEcosystem, CanonicalPackage: freebsdRetentionKey}) -} - -// freebsdRetentionKey maps pkg:bsd/freebsd/{name} to the stored package PURL. -// Another namespace, or a name with a slash, can never match a stored package -// and is refused. -func freebsdRetentionKey(p *purl.PURL) (string, bool) { - if p == nil || p.Type != "bsd" || p.Namespace != freebsdNamespace || strings.Contains(p.Name, "/") { - return "", false - } - canonical := purl.MakePURLString(freebsdEcosystem, p.Name, "") - return canonical, canonical != "" -} - -// FreeBSDHandler serves FreeBSD pkg repositories, ports and pkgbase alike. -// Each configured upstream is mounted at /freebsd/{repository}/ and the rest -// of the path mirrors the upstream layout ({ABI}/{repo}/{file}). -// -// Repository catalogues (meta.conf, packagesite.pkg, data.pkg) carry their -// signature inside the archive, so they are served byte for byte through the -// metadata cache. Packages under Hashed/ have the content hash in their name -// and never change, so they go to the artifact cache. Other package files, -// unhashed ones and the bootstrap package under Latest/, can be rebuilt under -// the same name and are passed through after the denylist; while a scanner -// covers freebsd they are refused, since only cached artifacts are scanned. -// Everything else, such as signatures and keys, is passed through. -type FreeBSDHandler struct { - proxy *Proxy - proxyURL string - repositories map[string]string -} - -// NewFreeBSDHandler creates a FreeBSD pkg repository handler. Without -// repositories it serves the official mirror as "freebsd". -func NewFreeBSDHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *FreeBSDHandler { - h := &FreeBSDHandler{ - proxy: proxy, - proxyURL: strings.TrimSuffix(proxyURL, "/"), - repositories: make(map[string]string, len(repositories)), - } - for name, repositoryURL := range repositories { - h.repositories[name] = strings.TrimSuffix(repositoryURL, "/") - } - if len(h.repositories) == 0 { - h.repositories[defaultFreeBSDRepositoryName] = defaultFreeBSDUpstream - } - return h -} - -// Routes returns the HTTP handler for pkg requests. -// Mount this at /freebsd on your router. -func (h *FreeBSDHandler) Routes() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet && r.Method != http.MethodHead { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - - path := strings.TrimPrefix(r.URL.Path, "/") - - // A decoded '?' or '#' would cut the upstream path short and cache one - // file under another file's name. - if containsPathTraversal(path) || strings.ContainsAny(path, "?#") { - http.Error(w, "invalid path", http.StatusBadRequest) - return - } - - repository, rest, ok := strings.Cut(path, "/") - upstreamURL, found := h.repositories[repository] - if !ok || rest == "" || !found { - http.NotFound(w, r) - return - } - - switch { - case isFreeBSDCatalogue(rest): - h.handleMetadata(w, r, repository, upstreamURL, rest) - case isFreeBSDHashedPackage(rest): - h.handlePackageDownload(w, r, repository, upstreamURL, rest) - case isFreeBSDPackageFile(rest): - h.handleUncachedPackage(w, r, upstreamURL, rest) - default: - h.proxyFile(w, r, upstreamURL, rest) - } - }) -} - -// freebsdCataloguePattern matches the files pkg reads to update a repository: -// meta and meta.conf, and the packagesite and data catalogues in the current -// (.pkg) and older (.txz, .tzst) packing formats. -var freebsdCataloguePattern = regexp.MustCompile(`^(?:meta(?:\.conf|\.txz|\.tzst|\.pkg)?|(?:packagesite|data)\.(?:pkg|txz|tzst))$`) - -func freebsdBase(path string) string { - return path[strings.LastIndex(path, "/")+1:] -} - -// isFreeBSDCatalogue reports whether path names a repository catalogue. The -// depth isn't checked, so repositories built with poudriere and served under -// another layout work too; package files are always {name}-{version}.pkg and -// can't take these names. -func isFreeBSDCatalogue(path string) bool { - return freebsdCataloguePattern.MatchString(freebsdBase(path)) -} - -// freebsdHashedPattern matches {name}-{version}~{hash}.pkg. Port versions -// never contain '-', so the version is what follows the last one, e.g. -// zsh-navigation-tools-2.2.7~2$imxi7k7w.pkg or -// FreeBSD-syslogd-man-14.snap20261019150821~2$k6xtent4.pkg. -var freebsdHashedPattern = regexp.MustCompile(`^(.+)-([^-~/]+)~[^~/]+\.pkg$`) - -// isFreeBSDHashedPackage reports whether path names a package in a Hashed/ -// directory: All/Hashed/ for ports, Hashed/ for pkgbase. -func isFreeBSDHashedPackage(path string) bool { - dir := strings.TrimSuffix(path, "/"+freebsdBase(path)) - return freebsdBase(dir) == freebsdHashedDir && freebsdHashedPattern.MatchString(freebsdBase(path)) -} - -// parseFreeBSDPackage returns the name and version of a hashed package file. -func parseFreeBSDPackage(filename string) (name, version string) { - matches := freebsdHashedPattern.FindStringSubmatch(filename) - if len(matches) != freebsdMatchCount { - return "", "" - } - return matches[1], matches[2] -} - -// freebsdUnhashedPattern matches {name}-{version} package files in the current -// (.pkg) and older (.txz, .tzst) packing formats, split at the last '-' like -// freebsdHashedPattern. -var freebsdUnhashedPattern = regexp.MustCompile(`^(.+)-([^-~/]+)\.(?:pkg|txz|tzst)$`) - -// freebsdPackageFilePattern matches every package file name, including -// Latest/pkg.pkg, which has no version. Catalogues match it too and must be -// routed first. -var freebsdPackageFilePattern = regexp.MustCompile(`\.(?:pkg|txz|tzst)$`) - -// isFreeBSDPackageFile reports whether path names a package file. -func isFreeBSDPackageFile(path string) bool { - return freebsdPackageFilePattern.MatchString(freebsdBase(path)) -} - -// parseFreeBSDPackageFile returns the name and version of any package file, -// hashed or not, or empty strings for a file without a version such as -// Latest/pkg.pkg. -func parseFreeBSDPackageFile(filename string) (name, version string) { - if name, version = parseFreeBSDPackage(filename); name != "" { - return name, version - } - matches := freebsdUnhashedPattern.FindStringSubmatch(filename) - if len(matches) != freebsdMatchCount { - return "", "" - } - return matches[1], matches[2] -} - -// errFreeBSDUnscannable refuses package files that can't be cached while -// scanning covers freebsd. -var errFreeBSDUnscannable = errors.New("package file can't be scanned: only packages under Hashed/ are cached and scanned; build the repository with pkg repo --hash") - -// handleUncachedPackage passes a package file outside Hashed/ through. Its -// name may be reused for a rebuild with other bytes, which pkg would then -// reject against the catalogue checksum, so it is never cached. The denylist -// applies when the name carries a version. Scanners only see cached -// artifacts, so the file is refused while one covers freebsd. -func (h *FreeBSDHandler) handleUncachedPackage(w http.ResponseWriter, r *http.Request, upstreamURL, path string) { - name, version := parseFreeBSDPackageFile(freebsdBase(path)) - if name != "" && h.proxy.versionDenied(freebsdEcosystem, name, version) { - h.proxy.serveArtifactError(w, fmt.Errorf("%w: %s", ErrVersionDenied, canonicalVersionPURL(freebsdEcosystem, name, version)), "") - return - } - if h.proxy.Scanners.Covers(freebsdEcosystem) { - h.proxy.Logger.Warn("refusing unscannable freebsd package", "path", path) - JSONError(w, http.StatusForbidden, errFreeBSDUnscannable.Error()) - return - } - h.proxyFile(w, r, upstreamURL, path) -} - -// handlePackageDownload caches hashed packages. The cache key holds the -// repository and full path, since every ABI and repository has its own -// builds. -func (h *FreeBSDHandler) handlePackageDownload(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - name, version := parseFreeBSDPackage(freebsdBase(path)) - if name == "" { - h.proxyFile(w, r, upstreamURL, path) - return - } - - downloadURL := upstreamURL + "/" + path - cacheFilename := repository + "/" + path - - h.proxy.Logger.Info("freebsd package download", - "repository", repository, "name", name, "version", version) - - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), freebsdEcosystem, name, version, cacheFilename, downloadURL) - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") - return - } - - if result.Artifact.MediaType == "" { - result.Artifact.MediaType = freebsdPackageMedia - } - ServeArtifactRequest(w, r, result) -} - -// handleMetadata serves catalogues unchanged through the metadata cache, -// which keeps their embedded signatures valid. -func (h *FreeBSDHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, freebsdEcosystem, - h.metadataCacheKey(repository, upstreamURL, path), "*/*") -} - -// metadataCacheKey hashes repository, upstream URL and path, as APKHandler does. -func (h *FreeBSDHandler) metadataCacheKey(repository, upstreamURL, path string) string { - identity := repository + "\x00" + upstreamURL + "\x00" + path - digest := sha256.Sum256([]byte(identity)) - return hex.EncodeToString(digest[:]) -} - -// proxyFile passes other files through uncached. -func (h *FreeBSDHandler) proxyFile(w http.ResponseWriter, r *http.Request, upstreamURL, path string) { - h.proxy.ProxyFile(w, r, upstreamURL+"/"+path) -} diff --git a/internal/handler/freebsd_test.go b/internal/handler/freebsd_test.go deleted file mode 100644 index 4702f34..0000000 --- a/internal/handler/freebsd_test.go +++ /dev/null @@ -1,407 +0,0 @@ -package handler - -import ( - "io" - "log/slog" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/scanner" - "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries/fetch" -) - -const freebsdABI = "/FreeBSD:14:amd64" - -func TestFreeBSDPathClassification(t *testing.T) { - tests := []struct { - path string - catalogue bool - hashed bool - }{ - {"FreeBSD:14:amd64/latest/meta.conf", true, false}, - {"FreeBSD:14:amd64/latest/meta", true, false}, - {"FreeBSD:14:amd64/latest/packagesite.pkg", true, false}, - {"FreeBSD:14:amd64/latest/data.pkg", true, false}, - {"FreeBSD:13:amd64/quarterly/packagesite.txz", true, false}, - {"142amd64-default/packagesite.tzst", true, false}, - {"FreeBSD:14:amd64/latest/All/Hashed/zsh-5.9_5~2$3rwh1rgp.pkg", false, true}, - {"FreeBSD:15:amd64/base_latest/Hashed/FreeBSD-acct-15.snap20261021214449~2$k6xtent4.pkg", false, true}, - {"FreeBSD:14:amd64/latest/All/zsh-5.9_5.pkg", false, false}, - {"FreeBSD:15:amd64/base_latest/FreeBSD-acct-15.snap20261021214449.pkg", false, false}, - {"FreeBSD:14:amd64/latest/Latest/pkg.pkg", false, false}, - {"FreeBSD:14:amd64/latest/Latest/pkg.pkg.sig", false, false}, - {"FreeBSD:14:amd64/latest/All/Hashed/no-hash.pkg", false, false}, - } - for _, tt := range tests { - if got := isFreeBSDCatalogue(tt.path); got != tt.catalogue { - t.Errorf("isFreeBSDCatalogue(%q) = %v, want %v", tt.path, got, tt.catalogue) - } - if got := isFreeBSDHashedPackage(tt.path); got != tt.hashed { - t.Errorf("isFreeBSDHashedPackage(%q) = %v, want %v", tt.path, got, tt.hashed) - } - } -} - -func TestParseFreeBSDPackage(t *testing.T) { - tests := []struct { - filename, name, version string - }{ - {"zsh-5.9_5~2$3rwh1rgp.pkg", "zsh", "5.9_5"}, - {"zsh-navigation-tools-2.2.7~2$imxi7k7w.pkg", "zsh-navigation-tools", "2.2.7"}, - {"py311-requests-2.32.3_1,1~2$abcdefgh.pkg", "py311-requests", "2.32.3_1,1"}, - {"FreeBSD-syslogd-man-14.snap20261019150821~2$k6xtent4.pkg", "FreeBSD-syslogd-man", "14.snap20261019150821"}, - {"nohash-1.0.pkg", "", ""}, - } - for _, tt := range tests { - name, version := parseFreeBSDPackage(tt.filename) - if name != tt.name || version != tt.version { - t.Errorf("parseFreeBSDPackage(%q) = %q, %q, want %q, %q", tt.filename, name, version, tt.name, tt.version) - } - } -} - -func TestParseFreeBSDPackageFile(t *testing.T) { - tests := []struct { - filename, name, version string - pkgfile bool - }{ - {"zsh-5.9_5~2$3rwh1rgp.pkg", "zsh", "5.9_5", true}, - {"zsh-5.9_5.pkg", "zsh", "5.9_5", true}, - {"zsh-navigation-tools-2.2.7.pkg", "zsh-navigation-tools", "2.2.7", true}, - {"py311-requests-2.32.3_1,1.txz", "py311-requests", "2.32.3_1,1", true}, - {"FreeBSD-acct-15.snap20261021214449.tzst", "FreeBSD-acct", "15.snap20261021214449", true}, - {"pkg.pkg", "", "", true}, - {"pkg.txz", "", "", true}, - {"pkg.pkg.sig", "", "", false}, - {"pkg.pkg.pub", "", "", false}, - {"zsh-5.9_5.pkg.sig", "", "", false}, - {"zsh-5.9_5.tar", "", "", false}, - } - for _, tt := range tests { - name, version := parseFreeBSDPackageFile(tt.filename) - if name != tt.name || version != tt.version { - t.Errorf("parseFreeBSDPackageFile(%q) = %q, %q, want %q, %q", tt.filename, name, version, tt.name, tt.version) - } - if got := isFreeBSDPackageFile("FreeBSD:14:amd64/latest/All/" + tt.filename); got != tt.pkgfile { - t.Errorf("isFreeBSDPackageFile(%q) = %v, want %v", tt.filename, got, tt.pkgfile) - } - } -} - -func TestFreeBSDHandler_Routes(t *testing.T) { - h := NewFreeBSDHandler(nil, "http://localhost:8080", nil) - assertRoutesBasics(t, h.Routes(), "/freebsd"+freebsdABI+"/latest/meta.conf", "/freebsd"+freebsdABI+"/../../../etc/passwd") -} - -func TestFreeBSDHandler_DefaultsToOfficialMirror(t *testing.T) { - h := NewFreeBSDHandler(nil, "http://localhost:8080", nil) - if got := h.repositories[defaultFreeBSDRepositoryName]; got != defaultFreeBSDUpstream { - t.Errorf("default repository = %q, want %q", got, defaultFreeBSDUpstream) - } -} - -func TestFreeBSDHandler_UnknownRepositoryReturns404(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - h := NewFreeBSDHandler(proxy, "http://localhost:8080", map[string]string{"freebsd": "https://example.test"}) - for _, target := range []string{"/unknown" + freebsdABI + "/latest/meta.conf", "/freebsd", "/"} { - if w := serveFreeBSDRequest(h, target); w.Code != http.StatusNotFound { - t.Errorf("%s: status = %d, want 404", target, w.Code) - } - } -} - -// fakeFreeBSDMirror serves files and counts requests per path. It answers -// 503 for everything while down is set. -type fakeFreeBSDMirror struct { - *httptest.Server - down atomic.Bool - mu sync.Mutex - requests map[string]int -} - -func newFakeFreeBSDMirror(t *testing.T, files map[string]string) *fakeFreeBSDMirror { - t.Helper() - m := &fakeFreeBSDMirror{requests: map[string]int{}} - m.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if m.down.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - body, ok := files[r.URL.Path] - if !ok { - http.NotFound(w, r) - return - } - m.mu.Lock() - m.requests[r.URL.Path]++ - m.mu.Unlock() - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = w.Write([]byte(body)) - })) - t.Cleanup(m.Close) - return m -} - -func (m *fakeFreeBSDMirror) count(path string) int { - m.mu.Lock() - defer m.mu.Unlock() - return m.requests[path] -} - -func TestFreeBSDHandler_CataloguesServedUnchanged(t *testing.T) { - files := map[string]string{ - freebsdABI + "/latest/meta.conf": "version = 2;\npacking_format = \"tzst\";\n", - freebsdABI + "/latest/packagesite.pkg": "\x28\xb5\x2f\xfdpackagesite.yaml and its signature", - freebsdABI + "/latest/data.pkg": "\x28\xb5\x2f\xfddata and its signature", - } - mirror := newFakeFreeBSDMirror(t, files) - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = mirror.Client() - h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) - - for round, down := range []bool{false, true} { - mirror.down.Store(down) - for path, want := range files { - w := serveFreeBSDRequest(h, "/freebsd"+path) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Fatalf("round %d %s: status = %d, body %q, want 200 and the upstream bytes", round, path, w.Code, w.Body) - } - } - } - for path := range files { - if got := mirror.count(path); got != 1 { - t.Errorf("%s fetched %d times, want 1", path, got) - } - } -} - -func TestFreeBSDHandler_HashedPackagesCachedPerABI(t *testing.T) { - const file = "/latest/All/Hashed/zsh-5.9_5~2$3rwh1rgp.pkg" - files := map[string]string{ - "/FreeBSD:14:amd64" + file: "amd64 package", - "/FreeBSD:14:aarch64" + file: "aarch64 package", - } - mirror := newFakeFreeBSDMirror(t, files) - proxy, db, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(mirror.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) - - for round, down := range []bool{false, true} { - mirror.down.Store(down) - for path, want := range files { - w := serveFreeBSDRequest(h, "/freebsd"+path) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Fatalf("round %d %s: status = %d, body %q, want 200 and %q", round, path, w.Code, w.Body, want) - } - } - } - for path := range files { - if got := mirror.count(path); got != 1 { - t.Errorf("%s fetched %d times, want 1", path, got) - } - } - - record, err := db.GetArtifact("pkg:bsd/freebsd/zsh@5.9_5", "freebsd/FreeBSD:14:amd64"+file) - if err != nil || record == nil { - t.Errorf("artifact record for pkg:bsd/freebsd/zsh@5.9_5 = %v, %v", record, err) - } -} - -// Unhashed packages can be rebuilt under the same name, and the bootstrap -// files under Latest/ change with every build, so none of them are cached. -// Without a denylist entry or a scanner they are passed through. -func TestFreeBSDHandler_UnhashedFilesPassedThrough(t *testing.T) { - files := map[string]string{ - freebsdABI + "/latest/Latest/pkg.pkg": "bootstrap package", - freebsdABI + "/latest/Latest/pkg.pkg.sig": "bootstrap signature", - freebsdABI + "/latest/All/zsh-5.9_5.pkg": "unhashed package", - } - mirror := newFakeFreeBSDMirror(t, files) - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = mirror.Client() - h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) - - for range 2 { - for path, want := range files { - w := serveFreeBSDRequest(h, "/freebsd"+path) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Fatalf("%s: status = %d, body %q, want 200 and %q", path, w.Code, w.Body, want) - } - } - } - for path := range files { - if got := mirror.count(path); got != 2 { - t.Errorf("%s fetched %d times, want 2 (not cached)", path, got) - } - } -} - -// Package files without a hash in their name are checked against the -// denylist like hashed ones, before anything is fetched. -func TestFreeBSDHandler_DenylistCoversEveryPackageFile(t *testing.T) { - denied := []string{ - freebsdABI + "/latest/All/Hashed/zsh-5.9_5~2$3rwh1rgp.pkg", - freebsdABI + "/latest/All/zsh-5.9_5.pkg", - freebsdABI + "/latest/All/zsh-5.9_5~2$3rwh1rgp.pkg", - "/142amd64-default/All/zsh-5.9_5.txz", - "/142amd64-default/All/zsh-5.9_5.tzst", - } - allowed := map[string]string{ - freebsdABI + "/latest/All/zsh-5.9_4.pkg": "older build", - freebsdABI + "/latest/All/zsh-navigation-tools-5.9_5.pkg": "another package", - } - files := map[string]string{} - for _, path := range denied { - files[path] = "denied package" - } - for path, body := range allowed { - files[path] = body - } - mirror := newFakeFreeBSDMirror(t, files) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = mirror.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(mirror.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - setTestDenylist(t, proxy, "pkg:bsd/freebsd/zsh@5.9_5") - h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) - - for _, path := range denied { - if w := serveFreeBSDRequest(h, "/freebsd"+path); w.Code != http.StatusForbidden { - t.Errorf("%s: status = %d, want 403", path, w.Code) - } - if got := mirror.count(path); got != 0 { - t.Errorf("%s fetched %d times, want 0", path, got) - } - } - for path, want := range allowed { - if w := serveFreeBSDRequest(h, "/freebsd"+path); w.Code != http.StatusOK || w.Body.String() != want { - t.Errorf("%s: status = %d, body %q, want 200 and %q", path, w.Code, w.Body, want) - } - } -} - -// Package files the proxy can't cache can't be scanned either, so they are -// refused while a scanner covers freebsd. Catalogues, signatures and hashed -// packages are unaffected. -func TestFreeBSDHandler_ScanningRefusesUncachablePackages(t *testing.T) { - refused := []string{ - freebsdABI + "/latest/All/zsh-5.9_5.pkg", - freebsdABI + "/latest/Latest/pkg.pkg", - "/142amd64-default/Latest/pkg.txz", - "/142amd64-default/All/zsh-5.9_5.tzst", - } - served := map[string]string{ - freebsdABI + "/latest/meta.conf": "version = 2;\n", - freebsdABI + "/latest/Latest/pkg.pkg.sig": "bootstrap signature", - freebsdABI + "/latest/Latest/pkg.pkg.pub": "bootstrap key", - } - files := map[string]string{} - for _, path := range refused { - files[path] = "unscanned package" - } - for path, body := range served { - files[path] = body - } - - for _, ecosystems := range [][]string{nil, {"freebsd"}} { - mirror := newFakeFreeBSDMirror(t, files) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = mirror.Client() - proxy.Scanners = newFreeBSDScanGroup(t, ecosystems) - h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) - - for _, path := range refused { - w := serveFreeBSDRequest(h, "/freebsd"+path) - if w.Code != http.StatusForbidden || !strings.Contains(w.Body.String(), "scan") { - t.Errorf("ecosystems %v %s: status = %d, body %q, want 403 naming the scan", ecosystems, path, w.Code, w.Body) - } - if got := mirror.count(path); got != 0 { - t.Errorf("ecosystems %v %s fetched %d times, want 0", ecosystems, path, got) - } - } - for path, want := range served { - if w := serveFreeBSDRequest(h, "/freebsd"+path); w.Code != http.StatusOK || w.Body.String() != want { - t.Errorf("ecosystems %v %s: status = %d, body %q, want 200 and %q", ecosystems, path, w.Code, w.Body, want) - } - } - } -} - -// A scanner restricted to other ecosystems never sees FreeBSD packages, so -// unhashed files are passed through as without scanning. -func TestFreeBSDHandler_ScannerForOtherEcosystemsPassesThrough(t *testing.T) { - const path = freebsdABI + "/latest/All/zsh-5.9_5.pkg" - mirror := newFakeFreeBSDMirror(t, map[string]string{path: "unhashed package"}) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = mirror.Client() - proxy.Scanners = newFreeBSDScanGroup(t, []string{"npm"}) - h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) - - if w := serveFreeBSDRequest(h, "/freebsd"+path); w.Code != http.StatusOK || w.Body.String() != "unhashed package" { - t.Errorf("status = %d, body %q, want 200 and the upstream bytes", w.Code, w.Body) - } -} - -func newFreeBSDScanGroup(t *testing.T, ecosystems []string) *scanner.Group { - t.Helper() - g, err := scanner.NewGroup(config.ScanningConfig{ - Enabled: true, - SigningKey: "test-signing-key", - Scanners: []config.ScannerConfig{ - {Name: "test-scanner", URL: "http://scanner.invalid/scan", Ecosystems: ecosystems}, - }, - }, slog.New(slog.NewTextHandler(io.Discard, nil))) - if err != nil { - t.Fatalf("scanner.NewGroup() error: %v", err) - } - return g -} - -func serveFreeBSDRequest(h *FreeBSDHandler, target string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - return w -} - -func TestFreeBSDRetentionKey(t *testing.T) { - tests := []struct { - key string - want string - ok bool - }{ - {"pkg:bsd/freebsd/zsh", "pkg:bsd/freebsd/zsh", true}, - {"pkg:bsd/FreeBSD/FreeBSD-acct", "pkg:bsd/freebsd/FreeBSD-acct", true}, - {"pkg:bsd/freebsd/libstdc++", "pkg:bsd/freebsd/libstdc%2B%2B", true}, - {"pkg:bsd/netbsd/zsh", "", false}, - {"pkg:bsd/zsh", "", false}, - {"pkg:bsd/freebsd/latest%2Fzsh", "", false}, - {"pkg:apk/freebsd/zsh", "", false}, - } - for _, tt := range tests { - p, err := purl.Parse(tt.key) - if err != nil { - t.Fatalf("Parse(%s): %v", tt.key, err) - } - got, ok := freebsdRetentionKey(p) - if ok != tt.ok || (ok && got != tt.want) { - t.Errorf("freebsdRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) - } - } - if _, ok := freebsdRetentionKey(nil); ok { - t.Error("freebsdRetentionKey(nil) accepted") - } -} diff --git a/internal/handler/gem.go b/internal/handler/gem.go index fa7c24b..9ec57e3 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -1,29 +1,21 @@ package handler import ( - "bytes" + "bufio" "encoding/json" "fmt" + "io" "net/http" - "regexp" "strings" "time" - "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( gemUpstream = "https://rubygems.org" ) -// Gems are stored under pkg:gem/{name} with the name from the gem filename, -// and purl keeps gem names as they are, so package rules use the default -// key mapping. The spec also claims the "rubygems" spelling git-pkgs writes -// into packages.ecosystem. -func init() { - retention.Default.Register(retention.Spec{Key: "gem", CanonicalPackage: retention.DefaultCanonical("gem")}) -} - // GemHandler handles RubyGems registry protocol requests. type GemHandler struct { proxy *Proxy @@ -40,13 +32,6 @@ func NewGemHandler(proxy *Proxy, proxyURL string) *GemHandler { } } -// NewGemHandlerWithUpstream creates a RubyGems handler with a custom upstream. -func NewGemHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *GemHandler { - h := NewGemHandler(proxy, proxyURL) - h.upstreamURL = configuredUpstreamURL(upstreamURL, gemUpstream) - return h -} - // Routes returns the HTTP handler for RubyGems requests. func (h *GemHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -75,205 +60,179 @@ func (h *GemHandler) Routes() http.Handler { // handleDownload serves a gem file, fetching and caching from upstream if needed. func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { - h.proxy.handleFilenameDownload(w, r, filenameDownload{ - ecosystem: "gem", - upstreamURL: h.upstreamURL, - suffix: ".gem", - parseErr: "could not parse gem filename", - fetchErr: "failed to fetch gem", - parse: h.parseGemFilename, - publishTime: h.publishTime, - }) + filename := r.PathValue("filename") + if filename == "" || !strings.HasSuffix(filename, ".gem") { + http.Error(w, "invalid filename", http.StatusBadRequest) + return + } + + // Extract name and version from filename (e.g., "rails-7.1.0.gem") + name, version := h.parseGemFilename(filename) + if name == "" || version == "" { + http.Error(w, "could not parse gem filename", http.StatusBadRequest) + return + } + + h.proxy.Logger.Info("gem download request", + "name", name, "version", version, "filename", filename) + + result, err := h.proxy.GetOrFetchArtifact(r.Context(), "gem", name, version, filename) + if err != nil { + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch gem", http.StatusBadGateway) + return + } + + ServeArtifact(w, result) } // parseGemFilename extracts name and version from a gem filename. // e.g., "rails-7.1.0.gem" -> ("rails", "7.1.0") // e.g., "aws-sdk-s3-1.142.0.gem" -> ("aws-sdk-s3", "1.142.0") -// e.g., "sorbet-static-0.5.11002-universal-darwin-22.gem" -> ("sorbet-static", "0.5.11002-universal-darwin-22") -// e.g., "devise-2fa-1.0.0.gem" -> ("devise-2fa", "1.0.0") -// -// A gem file is named {name}-{version}[-{platform}]. RubyGems writes a -// version as dot-separated alphanumeric segments starting with a digit, -// turning any "-" into ".pre.", and a platform starts with a letter (its -// CPU, or java), though it can end in a number (universal-java-17), so -// splitting at the last "-" would cut the platform. The version -// starts at the first "-" that leaves such a version and platform, -// so a name with a "-" segment of its own, such as devise-2fa, is -// kept whole. A filename no split fits is split at the first "-". -// -// Trying every "-" costs time quadratic in the filename's length, so -// a filename longer than maxGemFilenameLength is refused. func (h *GemHandler) parseGemFilename(filename string) (name, version string) { - if len(filename) > maxGemFilenameLength { - return "", "" - } + // Remove .gem extension base := strings.TrimSuffix(filename, ".gem") - first := -1 - for i := 0; i+1 < len(base); i++ { - if base[i] != '-' || base[i+1] < '0' || base[i+1] > '9' { - continue - } - if first < 0 { - first = i - } - if gemVersionAndPlatform(base[i+1:]) { + + // Find the last hyphen followed by a version number + // Version starts with a digit + for i := len(base) - 1; i >= 0; i-- { + if base[i] == '-' && i+1 < len(base) && base[i+1] >= '0' && base[i+1] <= '9' { return base[:i], base[i+1:] } } - if first < 0 { - return "", "" - } - return base[:first], base[first+1:] -} - -// maxGemFilenameLength bounds the gem filenames parseGemFilename accepts. -// Real gem filenames are a small fraction of this. -const maxGemFilenameLength = 512 - -var ( - gemVersionPattern = regexp.MustCompile(`^[0-9]+(\.[0-9A-Za-z]+)*$`) - gemPlatformPattern = regexp.MustCompile(`^[A-Za-z][0-9A-Za-z_]*(-[0-9A-Za-z_]+)*$`) -) - -// gemVersionAndPlatform reports whether s reads as {version}[-{platform}]. -func gemVersionAndPlatform(s string) bool { - version, platform, hasPlatform := strings.Cut(s, "-") - return gemVersionPattern.MatchString(version) && (!hasPlatform || gemPlatformPattern.MatchString(platform)) + return "", "" } // handleCompactIndex serves the compact index for a gem, filtering versions // based on cooldown when enabled. -// -// With cooldown the index and the versions API's publish times are read -// through the metadata cache, so a bundle install within metadata_ttl of the -// last one asks rubygems.org nothing, and one after it revalidates both with -// their ETags. The filtered index is kept in the rewrite cache and served -// with an ETag of its own. func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + h.proxyCached(w, r) + return + } + name := r.PathValue("name") if name == "" { http.Error(w, "invalid gem name", http.StatusBadRequest) return } - if !h.proxy.cooldownApplies("gem", name) { - h.proxyCached(w, r) - return - } h.proxy.Logger.Info("gem compact index request with cooldown", "name", name) - docs, err := h.proxy.fetchTimedMetadata(r.Context(), h.timedIndex(name), h.indexVersions) - switch { - case docs.doc.err != nil: - h.proxy.writeMetadataFetchError(w, docs.doc.err) - return - case docs.times.err != nil: - h.proxy.writeCooldownLookupError(w, "gem", name, docs.times.err) - return - case err != nil: - h.proxy.Logger.Error("failed to read gem compact index", "gem", name, "error", err) - http.Error(w, "failed to read upstream response", http.StatusBadGateway) - return - } - - packagePURL := canonicalPackagePURL("gem", name) - m, err := h.proxy.cachedRewrite(r.Context(), "gem", h.proxyURL, "info/"+name, docs.doc.body, docs.rewriteDigest(), - func([]byte) ([]byte, []string, error) { - body, err := h.proxy.decodedMetadataBody(docs.doc) - if err != nil { - return nil, nil, err - } - out, kept := h.filterIndex(body, name, func(version string) bool { - return docs.withheld(h.proxy, "gem", packagePURL, version) - }) - return out, kept, nil - }) + indexResp, filteredVersions, err := h.fetchIndexAndVersions(r, name) if err != nil { - h.proxy.Logger.Error("failed to filter gem compact index", "gem", name, "error", err) - http.Error(w, "failed to read upstream response", http.StatusBadGateway) + h.proxy.Logger.Error("upstream compact index request failed", "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + defer func() { _ = indexResp.Body.Close() }() + + if indexResp.StatusCode != http.StatusOK { + copyResponseHeaders(w, indexResp.Header) + w.WriteHeader(indexResp.StatusCode) + _, _ = io.Copy(w, indexResp.Body) return } - // Served as identity, as upstream's index was: Bundler checks digest - // headers against the bytes it receives, and asks for ranges of it. - m = m.identity() - contentType := docs.doc.contentType - if contentType == "" || contentType == contentTypeJSON { - contentType = "text/plain; charset=utf-8" + if filteredVersions == nil { + h.proxy.Logger.Warn("failed to fetch version timestamps, proxying unfiltered", "name", name) + copyResponseHeaders(w, indexResp.Header) + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, indexResp.Body) + return } - markStale(w, docs.doc.stale || docs.times.stale) - serveRewrittenMetadataAs(w, r, m, contentType) + + h.writeFilteredIndex(w, indexResp, name, filteredVersions) } -// timedIndex describes a gem's compact index and the versions API it takes -// publish times from. -func (h *GemHandler) timedIndex(name string) timedMetadata { - return timedMetadata{ - ecosystem: "gem", - docKey: "info/" + name, - docURL: h.upstreamURL + "/info/" + name, - timesKey: "api/v1/versions/" + name + ".json", - timesURL: fmt.Sprintf("%s/api/v1/versions/%s.json", h.upstreamURL, name), - timesAccept: contentTypeJSON, - parseTimes: parseGemVersions, +// fetchIndexAndVersions fetches the compact index and versions API concurrently. +// Returns the index response, a set of versions to filter (nil if versions API failed), +// and an error if the index fetch itself failed. +func (h *GemHandler) fetchIndexAndVersions(r *http.Request, name string) (*http.Response, map[string]bool, error) { + type versionsResult struct { + filtered map[string]bool + err error } + + versionsCh := make(chan versionsResult, 1) + go func() { + filtered, err := h.fetchFilteredVersions(r, name) + versionsCh <- versionsResult{filtered: filtered, err: err} + }() + + indexResp, err := h.fetchCompactIndex(r, name) + + versionsRes := <-versionsCh + + if err != nil { + return nil, nil, err + } + + if versionsRes.err != nil { + return indexResp, nil, nil + } + + return indexResp, versionsRes.filtered, nil } -// indexVersions lists the versions in a compact index, as the versions API -// names them. -func (h *GemHandler) indexVersions(doc metadataResult) ([]string, error) { - body, err := h.proxy.decodedMetadataBody(doc) +// fetchCompactIndex fetches the compact index from upstream. +func (h *GemHandler) fetchCompactIndex(r *http.Request, name string) (*http.Response, error) { + indexURL := h.upstreamURL + "/info/" + name + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, indexURL, nil) if err != nil { return nil, err } - var versions []string - for rest := body; len(rest) > 0; { - line, tail, _ := bytes.Cut(rest, []byte("\n")) - rest = tail - if version := gemIndexLineVersion(line); gemIndexListsVersion(version) { - versions = append(versions, string(version)) + for _, hdr := range []string{"Accept", headerAcceptEncoding, "If-None-Match", "If-Modified-Since"} { + if v := r.Header.Get(hdr); v != "" { + req.Header.Set(hdr, v) } } - return versions, nil + return h.proxy.HTTPClient.Do(req) } -// gemIndexLineVersion returns the version a compact index line starts with. -func gemIndexLineVersion(line []byte) []byte { - if spaceIdx := bytes.IndexByte(line, ' '); spaceIdx > 0 { - return line[:spaceIdx] - } - return line -} - -// gemIndexListsVersion reports whether a compact index line's first field -// is a version, rather than the "---" that starts the file or an empty line. -func gemIndexListsVersion(field []byte) bool { - return len(field) > 0 && string(field) != "---" -} - -// filterIndex returns the compact index without the lines of versions -// withheld reports, and the versions it kept. -func (h *GemHandler) filterIndex(body []byte, name string, withheld func(string) bool) ([]byte, []string) { - var kept []string - var out bytes.Buffer - out.Grow(len(body)) - for rest := body; len(rest) > 0; { - line, tail, _ := bytes.Cut(rest, []byte("\n")) - rest = tail - - version := gemIndexLineVersion(line) - if gemIndexListsVersion(version) { - if withheld(string(version)) { - h.proxy.Logger.Info("cooldown: filtering gem version", - "gem", name, "version", string(version)) - continue - } - kept = append(kept, string(version)) +// writeFilteredIndex writes the compact index response with cooldown-filtered versions removed. +func (h *GemHandler) writeFilteredIndex(w http.ResponseWriter, resp *http.Response, name string, filtered map[string]bool) { + for k, vv := range resp.Header { + if strings.EqualFold(k, "Content-Length") { + continue // length will change after filtering + } + for _, v := range vv { + w.Header().Add(k, v) + } + } + w.WriteHeader(http.StatusOK) + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + + if line == "---" { + _, _ = fmt.Fprintln(w, line) + continue + } + + version := line + if spaceIdx := strings.IndexByte(line, ' '); spaceIdx > 0 { + version = line[:spaceIdx] + } + + if filtered[version] { + h.proxy.Logger.Info("cooldown: filtering gem version", + "gem", name, "version", version) + continue + } + + _, _ = fmt.Fprintln(w, line) + } +} + +// copyResponseHeaders copies HTTP headers from a response to a writer. +func copyResponseHeaders(w http.ResponseWriter, headers http.Header) { + for k, vv := range headers { + for _, v := range vv { + w.Header().Add(k, v) } - out.Write(line) - out.WriteByte('\n') } - return out.Bytes(), kept } // gemVersion represents a version entry from the RubyGems versions API. @@ -283,30 +242,50 @@ type gemVersion struct { CreatedAt string `json:"created_at"` } -// parseGemVersions reads the publish times from a versions API response, -// keyed by version as the compact index writes it. -func parseGemVersions(body []byte) (publishTimes, error) { - var versions []gemVersion - if err := json.Unmarshal(body, &versions); err != nil { +// fetchFilteredVersions fetches the versions API and returns a set of version +// strings that should be filtered out by cooldown. +func (h *GemHandler) fetchFilteredVersions(r *http.Request, name string) (map[string]bool, error) { + versionsURL := fmt.Sprintf("%s/api/v1/versions/%s.json", h.upstreamURL, name) + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, versionsURL, nil) + if err != nil { return nil, err } - times := make(publishTimes, len(versions)) + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("versions API returned %d", resp.StatusCode) + } + + var versions []gemVersion + if err := json.NewDecoder(resp.Body).Decode(&versions); err != nil { + return nil, err + } + + packagePURL := purl.MakePURLString("gem", name, "") + filtered := make(map[string]bool) + for _, v := range versions { - versionStr := v.Number - if v.Platform != "" && v.Platform != "ruby" { - versionStr = v.Number + "-" + v.Platform - } createdAt, err := time.Parse(time.RFC3339, v.CreatedAt) if err != nil { - createdAt = time.Time{} + continue } - times[versionStr] = createdAt - } - return times, nil -} -func (h *GemHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, bool, error) { - return h.proxy.lookupPublishTime(r.Context(), h.timedIndex(name), version, h.indexVersions) + if !h.proxy.Cooldown.IsAllowed("gem", packagePURL, createdAt) { + // Build version string matching compact index format + versionStr := v.Number + if v.Platform != "" && v.Platform != "ruby" { + versionStr = v.Number + "-" + v.Platform + } + filtered[versionStr] = true + } + } + + return filtered, nil } // proxyCached forwards a metadata request with caching. @@ -340,16 +319,21 @@ func (h *GemHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { resp, err := h.proxy.HTTPClient.Do(req) if err != nil { - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return } defer func() { _ = resp.Body.Close() }() - h.proxy.relayResponse(w, r, resp, nil) + // Copy response headers + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) } func init() { diff --git a/internal/handler/gem_test.go b/internal/handler/gem_test.go index ba62ef8..7d90946 100644 --- a/internal/handler/gem_test.go +++ b/internal/handler/gem_test.go @@ -1,7 +1,6 @@ package handler import ( - "compress/gzip" "encoding/json" "fmt" "log/slog" @@ -26,18 +25,6 @@ func TestGemParseFilename(t *testing.T) { {"aws-sdk-s3-1.142.0.gem", "aws-sdk-s3", "1.142.0"}, {"nokogiri-1.15.4-x86_64-linux.gem", "nokogiri", "1.15.4-x86_64-linux"}, {"activerecord-7.0.8.gem", "activerecord", "7.0.8"}, - {"sorbet-static-0.5.11002-universal-darwin-22.gem", "sorbet-static", "0.5.11002-universal-darwin-22"}, - {"jruby-openssl-0.14.2-universal-java-17.gem", "jruby-openssl", "0.14.2-universal-java-17"}, - {"google-protobuf-3.25.3-arm64-darwin-23.gem", "google-protobuf", "3.25.3-arm64-darwin-23"}, - {"nokogiri-1.16.0-x64-mingw-ucrt.gem", "nokogiri", "1.16.0-x64-mingw-ucrt"}, - {"rake-13.1.0.pre.1.gem", "rake", "13.1.0.pre.1"}, - {"devise-2fa-1.0.0.gem", "devise-2fa", "1.0.0"}, - {"devise-2fa-1.0.0-java.gem", "devise-2fa", "1.0.0-java"}, - {"foo-2-1.0.0.gem", "foo-2", "1.0.0"}, - {"foo-2-1.gem", "foo-2", "1"}, - {"foo-2-x86_64-linux.gem", "foo", "2-x86_64-linux"}, - {"odd-1_0.gem", "odd", "1_0"}, - {"no-version.gem", "", ""}, {"invalid", "", ""}, } @@ -50,31 +37,6 @@ func TestGemParseFilename(t *testing.T) { } } -func TestGemDownloadRefusesLongFilename(t *testing.T) { - // Every "-1-" could start a version, and each try reads to the "!" at - // the end before failing, so parsing this used to take minutes. - filename := strings.Repeat("-1-aaaaaa", 100_000) + "!.gem" - - h := &GemHandler{proxy: testProxy(), upstreamURL: "http://upstream.invalid"} - req := httptest.NewRequest(http.MethodGet, "/gems/x", nil) - req.SetPathValue("filename", "x"+filename) - w := httptest.NewRecorder() - - done := make(chan struct{}) - go func() { - h.handleDownload(w, req) - close(done) - }() - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("parsing a long gem filename did not finish within 5s") - } - if w.Code != http.StatusBadRequest { - t.Errorf("status = %d, want %d", w.Code, http.StatusBadRequest) - } -} - func TestGemCompactIndexCooldown(t *testing.T) { now := time.Now() oldTime := now.Add(-7 * 24 * time.Hour).Format(time.RFC3339) @@ -133,118 +95,6 @@ func TestGemCompactIndexCooldown(t *testing.T) { } } -// gemCooldownUpstream serves index as /info/testgem, gzipped when the -// request accepts gzip as rubygems.org does, with 1.0.0 published a week ago -// and 2.0.0 an hour ago. -func gemCooldownUpstream(t *testing.T, index string, headers http.Header) *httptest.Server { - t.Helper() - now := time.Now() - versionsJSON, _ := json.Marshal([]gemVersion{ - {Number: "1.0.0", Platform: "ruby", CreatedAt: now.Add(-7 * 24 * time.Hour).Format(time.RFC3339)}, - {Number: "2.0.0", Platform: "ruby", CreatedAt: now.Add(-time.Hour).Format(time.RFC3339)}, - }) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/info/testgem": - for k, vv := range headers { - w.Header()[k] = vv - } - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { - _, _ = w.Write([]byte(index)) - return - } - w.Header().Set("Content-Encoding", "gzip") - gz := gzip.NewWriter(w) - _, _ = gz.Write([]byte(index)) - _ = gz.Close() - case "/api/v1/versions/testgem.json": - _, _ = w.Write(versionsJSON) - default: - w.WriteHeader(http.StatusNotFound) - } - })) - t.Cleanup(upstream.Close) - return upstream -} - -func TestGemCompactIndexCooldownClientAcceptsGzip(t *testing.T) { - index := "---\n1.0.0 dep1:>= 1.0|checksum:abc123\n2.0.0 dep1:>= 1.0|checksum:def456\n" - upstream := gemCooldownUpstream(t, index, http.Header{ - "Etag": {`"upstream"`}, - "Repr-Digest": {"sha-256=:upstream:"}, - }) - - proxy := testProxy() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL} - - // Bundler sends Accept-Encoding: gzip by default. - req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) - req.Header.Set("Accept-Encoding", "gzip;q=1.0,deflate;q=0.6,identity;q=0.3") - req.SetPathValue("name", "testgem") - w := httptest.NewRecorder() - h.handleCompactIndex(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) - } - if ce := w.Header().Get("Content-Encoding"); ce != "" { - t.Errorf("Content-Encoding = %q, want none", ce) - } - want := "---\n1.0.0 dep1:>= 1.0|checksum:abc123\n" - if got := w.Body.String(); got != want { - t.Errorf("body = %q, want %q", got, want) - } - if v := w.Header().Get("Repr-Digest"); v != "" { - t.Errorf("Repr-Digest = %q, upstream's value must not describe the filtered index", v) - } - if v := w.Header().Get("ETag"); v == "" || v == `"upstream"` { - t.Errorf("ETag = %q, want one of the filtered index", v) - } -} - -func TestGemCompactIndexCooldownLongLine(t *testing.T) { - // A version line longer than bufio.Scanner's 64 KB default used to end - // the response early with a 200. - long := "1.0.0 " + strings.Repeat("dep:>= 1.0,", 10_000) + "|checksum:abc" - index := "---\n" + long + "\n2.0.0 |checksum:def\n3.0.0 |checksum:ghi\n" - upstream := gemCooldownUpstream(t, index, nil) - - proxy := testProxy() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL} - - req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) - req.SetPathValue("name", "testgem") - w := httptest.NewRecorder() - h.handleCompactIndex(w, req) - - want := "---\n" + long + "\n3.0.0 |checksum:ghi\n" - if w.Code != http.StatusOK || w.Body.String() != want { - t.Fatalf("status = %d, body of %d bytes; want 200 with %d bytes", w.Code, w.Body.Len(), len(want)) - } -} - -func TestGemCompactIndexCooldownOversizedIndex(t *testing.T) { - index := "---\n1.0.0 |checksum:abc\n" + strings.Repeat("9.9.9 |checksum:zzz\n", 100) - upstream := gemCooldownUpstream(t, index, nil) - - proxy := testProxy() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - proxy.MetadataMaxSize = 256 - h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL} - - req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) - req.SetPathValue("name", "testgem") - w := httptest.NewRecorder() - h.handleCompactIndex(w, req) - - if w.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want %d", w.Code, http.StatusBadGateway) - } -} - func TestGemCompactIndexCooldownWithPlatformVersion(t *testing.T) { now := time.Now() recentTime := now.Add(-1 * time.Hour).Format(time.RFC3339) @@ -343,16 +193,18 @@ func TestGemCompactIndexVersionsAPIFails(t *testing.T) { w := httptest.NewRecorder() h.handleCompactIndex(w, req) - // Without publish times the index can't be filtered, so it isn't served. - if w.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want %d", w.Code, http.StatusBadGateway) + // Should still return OK with unfiltered content + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } - if strings.Contains(w.Body.String(), "1.0.0") { - t.Error("served the unfiltered index when the versions API failed") + + body := w.Body.String() + if !strings.Contains(body, "1.0.0") { + t.Error("expected unfiltered content when versions API fails") } } -func TestGemParseVersions(t *testing.T) { +func TestGemFetchFilteredVersions(t *testing.T) { now := time.Now() oldTime := now.Add(-7 * 24 * time.Hour).Format(time.RFC3339) recentTime := now.Add(-1 * time.Hour).Format(time.RFC3339) @@ -361,52 +213,40 @@ func TestGemParseVersions(t *testing.T) { {Number: "1.0.0", Platform: "ruby", CreatedAt: oldTime}, {Number: "2.0.0", Platform: "ruby", CreatedAt: recentTime}, {Number: "2.0.0", Platform: "java", CreatedAt: recentTime}, - {Number: "3.0.0", Platform: "ruby", CreatedAt: "not a time"}, }) - proxy := testProxy() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - - times, err := parseGemVersions(versionsJSON) - if err != nil { - t.Fatal(err) - } - docs := timedDocuments{published: times} - purl := canonicalPackagePURL("gem", "testgem") - for version, want := range map[string]bool{"1.0.0": false, "2.0.0": true, "2.0.0-java": true, "3.0.0": false, "4.0.0": false} { - if got := docs.withheld(proxy, "gem", purl, version); got != want { - t.Errorf("withheld(%s) = %v, want %v", version, got, want) - } - } - - _ = fmt.Sprintf // silence unused import -} - -func TestGemCooldownVersionsSizeLimit(t *testing.T) { - versionsJSON, _ := json.Marshal([]gemVersion{ - {Number: "2.0.0", Platform: "ruby", CreatedAt: time.Now().Add(-time.Hour).Format(time.RFC3339)}, - }) upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/api/v1/versions/") { - _, _ = w.Write(versionsJSON) - _, _ = w.Write([]byte(strings.Repeat(" ", 4096))) - return - } - _, _ = w.Write([]byte("---\n2.0.0 |checksum:abc\n")) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(versionsJSON) })) defer upstream.Close() proxy := testProxy() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - proxy.MetadataMaxSize = 1024 - h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL} + proxy.Cooldown = &cooldown.Config{ + Default: "3d", + } + + h := &GemHandler{ + proxy: proxy, + upstreamURL: upstream.URL, + proxyURL: "http://proxy.local", + } req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) - req.SetPathValue("name", "testgem") - w := httptest.NewRecorder() - h.handleCompactIndex(w, req) - - if w.Code != http.StatusBadGateway { - t.Errorf("status = %d, want %d", w.Code, http.StatusBadGateway) + filtered, err := h.fetchFilteredVersions(req, "testgem") + if err != nil { + t.Fatal(err) } + + if filtered["1.0.0"] { + t.Error("version 1.0.0 should not be filtered (old enough)") + } + if !filtered["2.0.0"] { + t.Error("version 2.0.0 (ruby) should be filtered") + } + if !filtered["2.0.0-java"] { + t.Error("version 2.0.0-java should be filtered") + } + + _ = fmt.Sprintf // silence unused import } diff --git a/internal/handler/generic.go b/internal/handler/generic.go deleted file mode 100644 index 0e08140..0000000 --- a/internal/handler/generic.go +++ /dev/null @@ -1,335 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "net/http" - "regexp" - "strings" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/retention" -) - -const ( - genericEcosystem = "generic" - // genericAcceptAny is always sent for metadata so every client shares the - // same cached representation, regardless of its Accept header. - genericAcceptAny = "*/*" - // githubReleaseAssetMatchCount is the full match plus owner, repository, - // tag and asset filename. - githubReleaseAssetMatchCount = 5 -) - -// githubReleaseAssetPattern matches the path of a GitHub release asset -// download, {owner}/{repo}/releases/download/{tag}/{asset}. A tag pins the -// asset to one release, so these downloads are cached in the artifact cache -// and served without revalidation once fetched. -var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/download/([^/]+)/([^/]+)$`) - -func init() { - retention.Default.Register(retention.Spec{Key: genericEcosystem, CanonicalPackage: retention.DefaultCanonical(genericEcosystem)}) -} - -// builtinGenericArtifacts maps well-known download hosts to patterns for -// their version-pinned paths. They apply to any generic upstream configured -// with that base URL, after the upstream's own generic_artifacts. -var builtinGenericArtifacts = map[string][]string{ - // dist/{project dirs}/{version}/[binaries/]{file}, e.g. maven/maven-3 or maven/mvnd. - "https://archive.apache.org": { - `dist/(?P.+?)/(?P[0-9][^/]*)/(?:binaries/)?(?P[^/]+)`, - }, - "https://nodejs.org": { - `dist/(?Pv[0-9][^/]*)/(?P[^/]+)`, - }, - // The version is only in the filename: go1.23.2.linux-amd64.tar.gz. - "https://dl.google.com/go": { - `(?Pgo(?P[0-9][^/]*?)\.[a-z0-9]+-[a-z0-9]+\.(?:tar\.gz|zip)(?:\.sha256)?)`, - }, -} - -// GenericHandler proxies plain HTTP downloads from configured upstream base -// URLs. Each configured upstream is mounted at /generic/{name}/ and the -// remaining request path (and query string) is appended to the upstream URL. -// -// Only configured upstreams are reachable, so the proxy is not an open HTTP -// proxy. The handler is the caching layer behind tools that download from -// fixed URL shapes, such as mise's aqua backend fetching GitHub release -// assets, and is pointed at by URL-rewriting settings on the client. -// -// Release-asset paths ({owner}/{repo}/releases/download/{tag}/{asset}), and -// paths matching the upstream's built-in or configured artifact patterns, are -// version-pinned and cached in the shared artifact cache, so they keep being -// served when the upstream is unreachable. Every other path is served through -// the metadata cache: fresh within the metadata TTL, revalidated with the -// upstream's validators after that, and served stale when the upstream fails -// or refuses the request. That covers API responses such as -// api.github.com/repos/{owner}/{repo}/releases/tags/{tag}. -type GenericHandler struct { - proxy *Proxy - repositories map[string]string - artifactPatterns map[string][]*regexp.Regexp - // paths holds each upstream's upstream.generic_paths patterns; an - // upstream without any serves every path. - paths map[string][]string -} - -// NewGenericHandler creates a generic HTTP download proxy handler. -// artifactPatterns maps upstream names to upstream.generic_artifacts patterns; -// a pattern that does not compile is logged and skipped. -func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPatterns ...map[string][]string) *GenericHandler { - h := &GenericHandler{ - proxy: proxy, - repositories: make(map[string]string, len(repositories)), - artifactPatterns: make(map[string][]*regexp.Regexp), - } - for name, upstreamURL := range repositories { - h.repositories[name] = strings.TrimSuffix(upstreamURL, "/") - } - // The first matching pattern wins, so configured patterns go first. - if len(artifactPatterns) > 0 { - for name, patterns := range artifactPatterns[0] { - h.addArtifactPatterns(name, patterns) - } - } - for name, upstreamURL := range h.repositories { - h.addArtifactPatterns(name, builtinGenericArtifacts[upstreamURL]) - } - return h -} - -// WithPaths limits upstreams to the paths their upstream.generic_paths -// patterns match. Any other path answers 404. -func (h *GenericHandler) WithPaths(paths map[string][]string) *GenericHandler { - h.paths = paths - return h -} - -func (h *GenericHandler) addArtifactPatterns(name string, patterns []string) { - for _, pattern := range patterns { - re, err := config.CompileGenericArtifactPattern(pattern) - if err != nil { - h.proxy.Logger.Warn("skipping generic artifact pattern", - "repository", name, "pattern", pattern, "error", err) - continue - } - h.artifactPatterns[name] = append(h.artifactPatterns[name], re) - } -} - -// Routes returns the HTTP handler for generic download requests. -// Mount this at /generic on your router. -func (h *GenericHandler) Routes() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet && r.Method != http.MethodHead { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - - path := strings.TrimPrefix(r.URL.Path, "/") - - if containsPathTraversal(path) { - http.Error(w, "invalid path", http.StatusBadRequest) - return - } - - repository, rest, ok := strings.Cut(path, "/") - if repository == HostedGenericRepository && h.proxy.HostedEnabled { - h.handleHosted(w, r, rest) - return - } - upstreamURL, found := h.repositories[repository] - if !ok || rest == "" || !found || !config.GenericPathAllowed(h.paths[repository], rest) { - http.NotFound(w, r) - return - } - - if artifact, ok := h.parseArtifact(repository, rest); ok { - h.handleArtifact(w, r, repository, upstreamURL, rest, artifact) - return - } - - h.handleMetadata(w, r, repository, upstreamURL, rest) - }) -} - -// HostedGenericRepository is the /generic/{name}/ path that serves pushed -// generic files instead of an upstream while hosted packages are enabled. -const HostedGenericRepository = config.HostedGenericRepository - -// handleHosted serves a pushed file at {name}/{version}/{file}. Pushed files -// never come from an upstream, so anything not pushed is a 404. -func (h *GenericHandler) handleHosted(w http.ResponseWriter, r *http.Request, path string) { - segments := strings.Split(path, "/") - if len(segments) != hostedGenericPathSegments { - http.NotFound(w, r) - return - } - target, err := HostedTargetFor(genericEcosystem, segments[0], segments[1], segments[2]) - if err != nil { - http.NotFound(w, r) - return - } - result, err := h.proxy.GetLocalArtifact(r.Context(), genericEcosystem, target.Name, target.Version, target.Filename) - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to read pushed file") - return - } - if result == nil { - http.NotFound(w, r) - return - } - serveGenericArtifact(w, r, result) -} - -// hostedGenericPathSegments is {name}/{version}/{file}. -const hostedGenericPathSegments = 3 - -// genericArtifact is the identity of a version-pinned download. -type genericArtifact struct { - name string - version string - filename string -} - -// parseArtifact returns the identity of a version-pinned download path: one -// matching the upstream's artifact patterns, else a GitHub release asset. -func (h *GenericHandler) parseArtifact(repository, path string) (genericArtifact, bool) { - for _, pattern := range h.artifactPatterns[repository] { - if artifact, ok := matchArtifactPattern(pattern, repository, path); ok { - return artifact, true - } - } - if asset, ok := parseGitHubReleaseAsset(path); ok { - return genericArtifact{ - name: asset.owner + "/" + asset.repo, - version: asset.tag, - filename: asset.filename, - }, true - } - return genericArtifact{}, false -} - -// matchArtifactPattern extracts the artifact identity from a path matched by -// pattern. The version and file must be single, non-empty path segments; the -// name defaults to the upstream name. -func matchArtifactPattern(pattern *regexp.Regexp, repository, path string) (genericArtifact, bool) { - matches := pattern.FindStringSubmatch(path) - if matches == nil { - return genericArtifact{}, false - } - artifact := genericArtifact{ - name: repository, - version: matches[pattern.SubexpIndex("version")], - filename: matches[pattern.SubexpIndex("file")], - } - if i := pattern.SubexpIndex("name"); i >= 0 && matches[i] != "" { - artifact.name = matches[i] - } - if artifact.version == "" || artifact.filename == "" || - strings.Contains(artifact.version, "/") || strings.Contains(artifact.filename, "/") { - return genericArtifact{}, false - } - return artifact, true -} - -// githubReleaseAsset is the identity of a version-pinned release download. -type githubReleaseAsset struct { - owner string - repo string - tag string - filename string -} - -// parseGitHubReleaseAsset extracts the release identity from a path shaped -// like {owner}/{repo}/releases/download/{tag}/{asset}. -func parseGitHubReleaseAsset(path string) (githubReleaseAsset, bool) { - matches := githubReleaseAssetPattern.FindStringSubmatch(path) - if len(matches) != githubReleaseAssetMatchCount { - return githubReleaseAsset{}, false - } - return githubReleaseAsset{ - owner: matches[1], - repo: matches[2], - tag: matches[3], - filename: matches[4], - }, true -} - -// handleArtifact fetches and caches a version-pinned download in the artifact -// cache. The upstream, full path and query identify the cached bytes, since -// distinct downloads can have the same captured name, version and filename. -func (h *GenericHandler) handleArtifact(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, artifact genericArtifact) { - downloadURL := upstreamURL + "/" + path - if r.URL.RawQuery != "" { - downloadURL += "?" + r.URL.RawQuery - } - cacheFilename := repository + "/" + h.cacheKey(repository, upstreamURL, path, r.URL.RawQuery) + "/" + artifact.filename - - h.proxy.Logger.Info("generic artifact download", - "repository", repository, "name", artifact.name, "version", artifact.version, "filename", artifact.filename) - - result, err := h.legacyArtifact(r.Context(), repository, downloadURL, artifact) - if err == nil && result == nil { - result, err = h.proxy.GetOrFetchArtifactFromURL( - r.Context(), genericEcosystem, artifact.name, artifact.version, cacheFilename, downloadURL) - } - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch artifact") - return - } - serveGenericArtifact(w, r, result) -} - -// serveGenericArtifact serves a cached or pushed file, as a plain download -// when its media type is unknown. -func serveGenericArtifact(w http.ResponseWriter, r *http.Request, result *CacheResult) { - if result.Artifact.MediaType == "" { - result.Artifact.MediaType = "application/octet-stream" - } - ServeArtifactRequest(w, r, result) -} - -// legacyArtifact returns the artifact cached under the unhashed -// {repository}/{filename} key, provided its recorded upstream URL is -// downloadURL, so a different path or query sharing that key is not served. -func (h *GenericHandler) legacyArtifact(ctx context.Context, repository, downloadURL string, artifact genericArtifact) (*CacheResult, error) { - pkgPURL, versionPURL, err := packagePURLStrings(genericEcosystem, artifact.name, artifact.version) - if err != nil { - return nil, err - } - filename := repository + "/" + artifact.filename - record, err := h.proxy.DB.GetArtifact(versionPURL, filename) - if err != nil { - return nil, fmt.Errorf("checking artifact cache: %w", err) - } - if record == nil || record.UpstreamURL != downloadURL { - return nil, nil - } - return h.proxy.checkCache(ctx, pkgPURL, versionPURL, filename) -} - -// handleMetadata serves any other path through the metadata cache. The query -// string is forwarded and is part of the cache identity. A fixed Accept header -// keeps all clients on one cached representation. -func (h *GenericHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - target := upstreamURL + "/" + path - if r.URL.RawQuery != "" { - target += "?" + r.URL.RawQuery - } - - h.proxy.ProxyCached(w, r, target, genericEcosystem, - h.cacheKey(repository, upstreamURL, path, r.URL.RawQuery), genericAcceptAny) -} - -// cacheKey derives the cache key from the upstream name, its -// URL, the request path and query. Hashing the identity keeps distinct -// upstreams from sharing entries and drops cached entries when an upstream is -// repointed, mirroring APKHandler.metadataCacheKey. -func (h *GenericHandler) cacheKey(repository, upstreamURL, path, query string) string { - identity := repository + "\x00" + upstreamURL + "\x00" + path + "\x00" + query - digest := sha256.Sum256([]byte(identity)) - return hex.EncodeToString(digest[:]) -} diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go deleted file mode 100644 index a74be8c..0000000 --- a/internal/handler/generic_test.go +++ /dev/null @@ -1,692 +0,0 @@ -package handler - -import ( - "context" - "io" - "net/http" - "net/http/httptest" - "regexp" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" - upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/registries/fetch" -) - -const testReleaseAssetPath = "/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" - -func TestGenericRetentionConfig(t *testing.T) { - cfg := config.Default() - cfg.Storage.Retention.Ecosystems = map[string]string{"generic": "30d"} - cfg.Storage.Retention.Packages = map[string]string{"pkg:generic/jqlang%2Fjq": "90d"} - - rules, err := cfg.RetentionRules(retention.Default) - if err != nil { - t.Fatalf("RetentionRules() error = %v", err) - } - if got := rules.For(genericEcosystem, "pkg:generic/jqlang%2Fjq"); got != 90*24*time.Hour { - t.Errorf("jqlang/jq retention = %v, want 90d", got) - } - if got := rules.For(genericEcosystem, "pkg:generic/other%2Frepo"); got != 30*24*time.Hour { - t.Errorf("generic ecosystem retention = %v, want 30d", got) - } -} - -func TestParseGitHubReleaseAsset(t *testing.T) { - tests := []struct { - path string - want githubReleaseAsset - ok bool - }{ - { - "jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64", - githubReleaseAsset{owner: "jqlang", repo: "jq", tag: "jq-1.7.1", filename: "jq-linux-amd64"}, - true, - }, - { - "cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.tar.gz", - githubReleaseAsset{owner: "cli", repo: "cli", tag: "v2.63.2", filename: "gh_2.63.2_linux_amd64.tar.gz"}, - true, - }, - // Mutable: resolves to whatever is latest today. - {"jqlang/jq/releases/latest/download/jq-linux-amd64", githubReleaseAsset{}, false}, - // API lookups and tag listings are not assets. - {"repos/jqlang/jq/releases/tags/jq-1.7.1", githubReleaseAsset{}, false}, - {"jqlang/jq/releases/tag/jq-1.7.1", githubReleaseAsset{}, false}, - // Source archives are a different shape. - {"jqlang/jq/archive/refs/tags/jq-1.7.1.tar.gz", githubReleaseAsset{}, false}, - // Extra or missing segments. - {"jqlang/jq/releases/download/jq-1.7.1", githubReleaseAsset{}, false}, - {"jqlang/jq/releases/download/jq-1.7.1/dir/asset", githubReleaseAsset{}, false}, - {"", githubReleaseAsset{}, false}, - } - - for _, tt := range tests { - got, ok := parseGitHubReleaseAsset(tt.path) - if ok != tt.ok || got != tt.want { - t.Errorf("parseGitHubReleaseAsset(%q) = (%+v, %v), want (%+v, %v)", tt.path, got, ok, tt.want, tt.ok) - } - } -} - -const testMavenPattern = `dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)` - -func TestMatchArtifactPattern(t *testing.T) { - named, err := config.CompileGenericArtifactPattern(testMavenPattern) - if err != nil { - t.Fatal(err) - } - unnamed, err := config.CompileGenericArtifactPattern(`dist/v(?P[^/]+)/(?P.+)`) - if err != nil { - t.Fatal(err) - } - - tests := []struct { - name string - pattern *regexp.Regexp - path string - want genericArtifact - ok bool - }{ - { - "name group", - named, - "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", - genericArtifact{name: "maven", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz"}, - true, - }, - { - "name defaults to upstream", - unnamed, - "dist/v20.15.0/node-v20.15.0-linux-x64.tar.xz", - genericArtifact{name: "apache", version: "20.15.0", filename: "node-v20.15.0-linux-x64.tar.xz"}, - true, - }, - // Patterns match whole paths only. - {"prefix only", named, "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz/extra", genericArtifact{}, false}, - {"suffix only", named, "mirror/dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", genericArtifact{}, false}, - {"directory listing", named, "dist/maven/maven-3/3.9.9/binaries/", genericArtifact{}, false}, - // A file group spanning segments would nest storage paths. - {"file with slash", unnamed, "dist/v20.15.0/win-x64/node.exe", genericArtifact{}, false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, ok := matchArtifactPattern(tt.pattern, "apache", tt.path) - if ok != tt.ok || got != tt.want { - t.Errorf("matchArtifactPattern(%q) = (%+v, %v), want (%+v, %v)", tt.path, got, ok, tt.want, tt.ok) - } - }) - } -} - -func TestGenericHandler_BuiltinArtifactPatterns(t *testing.T) { - h := NewGenericHandler(testProxy(), map[string]string{ - "apache": "https://archive.apache.org/", - "node": "https://nodejs.org", - "golang": "https://dl.google.com/go", - "mirror": "https://mirror.example.com", - }, map[string][]string{ - "apache": {`extra/(?P[^/]+)/(?P[^/]+)`, testMavenPattern}, - }) - - tests := []struct { - repository string - path string - want genericArtifact - ok bool - }{ - // The configured testMavenPattern overlaps the built-in and takes precedence. - {"apache", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", - genericArtifact{name: "maven", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz"}, true}, - {"apache", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz.sha512", - genericArtifact{name: "maven", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz.sha512"}, true}, - {"apache", "dist/maven/maven-4/4.0.0/binaries/apache-maven-4.0.0-bin.tar.gz", - genericArtifact{name: "maven/maven-4", version: "4.0.0", filename: "apache-maven-4.0.0-bin.tar.gz"}, true}, - {"apache", "dist/maven/mvnd/1.0.6/maven-mvnd-1.0.6-linux-amd64.tar.gz", - genericArtifact{name: "maven/mvnd", version: "1.0.6", filename: "maven-mvnd-1.0.6-linux-amd64.tar.gz"}, true}, - {"apache", "dist/maven/maven-3/", genericArtifact{}, false}, - {"apache", "dist/maven/maven-3/3.9.9/source/apache-maven-3.9.9-src.tar.gz", genericArtifact{}, false}, - {"apache", "extra/1.0/file.txt", genericArtifact{name: "apache", version: "1.0", filename: "file.txt"}, true}, - {"node", "dist/v22.11.0/node-v22.11.0-linux-x64.tar.gz", - genericArtifact{name: "node", version: "v22.11.0", filename: "node-v22.11.0-linux-x64.tar.gz"}, true}, - {"node", "dist/v22.11.0/SHASUMS256.txt", - genericArtifact{name: "node", version: "v22.11.0", filename: "SHASUMS256.txt"}, true}, - {"node", "dist/index.json", genericArtifact{}, false}, - {"golang", "go1.23.2.linux-amd64.tar.gz", - genericArtifact{name: "golang", version: "1.23.2", filename: "go1.23.2.linux-amd64.tar.gz"}, true}, - {"golang", "go1.23.2.darwin-arm64.tar.gz.sha256", - genericArtifact{name: "golang", version: "1.23.2", filename: "go1.23.2.darwin-arm64.tar.gz.sha256"}, true}, - {"golang", "go1.24rc1.windows-amd64.zip", - genericArtifact{name: "golang", version: "1.24rc1", filename: "go1.24rc1.windows-amd64.zip"}, true}, - {"mirror", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", genericArtifact{}, false}, - } - - for _, tt := range tests { - t.Run(tt.repository+"/"+tt.path, func(t *testing.T) { - got, ok := h.parseArtifact(tt.repository, tt.path) - if ok != tt.ok || got != tt.want { - t.Errorf("parseArtifact(%q, %q) = (%+v, %v), want (%+v, %v)", tt.repository, tt.path, got, ok, tt.want, tt.ok) - } - }) - } -} - -func TestGenericHandler_ArtifactPatternIsCachedAndOtherPathsAreNot(t *testing.T) { - const ( - artifactPath = "/dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz" - listingPath = "/dist/maven/maven-3/3.9.9/binaries/" - ) - tarball := []byte("maven tarball bytes") - var available atomic.Bool - available.Store(true) - var artifactRequests, listingRequests atomic.Int32 - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - switch r.URL.Path { - case artifactPath: - artifactRequests.Add(1) - _, _ = w.Write(tarball) - case listingPath: - listingRequests.Add(1) - _, _ = w.Write([]byte("listing")) - default: - http.NotFound(w, r) - } - })) - defer upstream.Close() - - proxy, db, store, _ := setupTestProxy(t) - store.seekable = true - proxy.HTTPClient = upstream.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewGenericHandler(proxy, - map[string]string{"apache": upstream.URL}, - map[string][]string{"apache": {testMavenPattern}}) - - w := serveGenericRequest(h, "/apache"+artifactPath) - if w.Code != http.StatusOK || w.Body.String() != string(tarball) { - t.Fatalf("status = %d, body = %q, want 200 %q", w.Code, w.Body.String(), tarball) - } - - pkg, err := db.GetPackageByEcosystemName(genericEcosystem, "maven") - if err != nil || pkg == nil { - t.Fatalf("package generic/maven = %v, %v; want a cached package", pkg, err) - } - versions, err := db.GetVersionsByPackagePURL(pkg.PURL) - if err != nil || len(versions) != 1 || !strings.HasSuffix(versions[0].PURL, "@3.9.9") { - t.Fatalf("versions = %+v, %v; want one @3.9.9", versions, err) - } - - available.Store(false) - w = serveGenericRequest(h, "/apache"+artifactPath) - if w.Code != http.StatusOK || w.Body.String() != string(tarball) { - t.Fatalf("cached: status = %d, body = %q, want 200 %q", w.Code, w.Body.String(), tarball) - } - if got := artifactRequests.Load(); got != 1 { - t.Errorf("artifact upstream requests = %d, want 1", got) - } - - // Paths the pattern does not match keep going through the metadata route. - available.Store(true) - for range 2 { - serveGenericRequest(h, "/apache"+listingPath) - } - if got := listingRequests.Load(); got != 2 { - t.Errorf("listing upstream requests = %d, want 2 with metadata caching off", got) - } -} - -func TestGenericHandler_ConfiguredArtifactsKeepDistinctPathsAndQueries(t *testing.T) { - for _, tt := range []struct { - name string - targets []string - }{ - {"platform paths", []string{ - "/releases/tool/1.0.0/linux/tool.tar.gz", - "/releases/tool/1.0.0/darwin/tool.tar.gz", - }}, - {"query parameters", []string{ - "/releases/tool/1.0.0/linux/tool.tar.gz?token=example&variant=a%2Bb", - "/releases/tool/1.0.0/linux/tool.tar.gz?token=other&variant=a+b", - "/releases/tool/1.0.0/linux/tool.tar.gz", - }}, - } { - t.Run(tt.name, func(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - // Echo the exact target so dropped or changed queries fail too. - _, _ = w.Write([]byte(r.URL.RequestURI())) - })) - defer upstream.Close() - - proxy, _, store, _ := setupTestProxy(t) - store.seekable = true - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewGenericHandler(proxy, - map[string]string{"downloads": upstream.URL}, - map[string][]string{"downloads": {`releases/(?Ptool)/(?P[^/]+)/(linux|darwin)/(?P[^/]+)`}}) - - for pass := range 2 { - for _, target := range tt.targets { - w := serveGenericRequest(h, "/downloads"+target) - if w.Code != http.StatusOK || w.Body.String() != target { - t.Fatalf("pass %d, target %q: status = %d, body = %q; want 200 %q", - pass, target, w.Code, w.Body.String(), target) - } - } - upstream.Close() - } - if got := requests.Load(); got != int32(len(tt.targets)) { - t.Errorf("upstream requests = %d, want %d", got, len(tt.targets)) - } - }) - } -} - -func TestGenericHandler_RejectsUnknownUpstreamAndBadPaths(t *testing.T) { - h := NewGenericHandler(testProxy(), map[string]string{"github": "https://github.com"}) - - tests := []struct { - name string - method string - target string - want int - }{ - {"unknown upstream", http.MethodGet, "/gitlab/owner/repo/releases/download/v1/asset", http.StatusNotFound}, - {"missing path", http.MethodGet, "/github", http.StatusNotFound}, - {"missing path with slash", http.MethodGet, "/github/", http.StatusNotFound}, - {"traversal", http.MethodGet, "/github/../etc/passwd", http.StatusBadRequest}, - {"encoded traversal", http.MethodGet, "/github/%2e%2e/etc/passwd", http.StatusBadRequest}, - {"post", http.MethodPost, "/github/owner/repo/releases/download/v1/asset", http.StatusMethodNotAllowed}, - {"hosted files with hosting off", http.MethodGet, "/hosted/tools/1.2.0/tool.tar.gz", http.StatusNotFound}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(tt.method, tt.target, nil)) - if w.Code != tt.want { - t.Errorf("status = %d, want %d", w.Code, tt.want) - } - }) - } -} - -func TestGenericHandler_ReleaseAssetIsCachedAndServedWhenUpstreamDown(t *testing.T) { - asset := []byte("jq binary bytes") - var available atomic.Bool - available.Store(true) - var upstreamRequests atomic.Int32 - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - if r.URL.Path != testReleaseAssetPath { - http.NotFound(w, r) - return - } - upstreamRequests.Add(1) - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = w.Write(asset) - })) - defer upstream.Close() - - proxy, db, store, _ := setupTestProxy(t) - store.seekable = true - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewGenericHandler(proxy, map[string]string{"github": upstream.URL}) - - w := serveGenericRequest(h, "/github"+testReleaseAssetPath) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - if got := w.Body.String(); got != string(asset) { - t.Errorf("body = %q, want %q", got, asset) - } - if got := upstreamRequests.Load(); got != 1 { - t.Fatalf("upstream requests = %d, want 1", got) - } - pkg, err := db.GetPackageByEcosystemName(genericEcosystem, "jqlang/jq") - if err != nil || pkg == nil { - t.Fatalf("release package = %v, %v; want a cached package", pkg, err) - } - if pkg.PURL != "pkg:generic/jqlang%2Fjq" { - t.Errorf("release package PURL = %q, want pkg:generic/jqlang%%2Fjq", pkg.PURL) - } - - // Second request must be served from cache, even with the upstream down. - available.Store(false) - rangeRequest := httptest.NewRequest(http.MethodGet, "/github"+testReleaseAssetPath, nil) - rangeRequest.Header.Set("Range", "bytes=0-2") - rangeResponse := httptest.NewRecorder() - h.Routes().ServeHTTP(rangeResponse, rangeRequest) - if rangeResponse.Code != http.StatusPartialContent { - t.Fatalf("range: status = %d, want 206: %s", rangeResponse.Code, rangeResponse.Body.String()) - } - if got := rangeResponse.Body.String(); got != string(asset[:3]) { - t.Errorf("range: body = %q, want %q", got, asset[:3]) - } - if got := rangeResponse.Header().Get("Content-Range"); got != "bytes 0-2/15" { - t.Errorf("range: Content-Range = %q, want %q", got, "bytes 0-2/15") - } - if got := upstreamRequests.Load(); got != 1 { - t.Errorf("upstream requests after range cache hit = %d, want 1", got) - } - - w = serveGenericRequest(h, "/github"+testReleaseAssetPath) - if w.Code != http.StatusOK { - t.Fatalf("cached: status = %d, want 200: %s", w.Code, w.Body.String()) - } - if got := w.Body.String(); got != string(asset) { - t.Errorf("cached: body = %q, want %q", got, asset) - } - if got := upstreamRequests.Load(); got != 1 { - t.Errorf("upstream requests after cache hit = %d, want 1", got) - } - - // HEAD is answered from the same cache entry without a body. - w = httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/github"+testReleaseAssetPath, nil)) - if w.Code != http.StatusOK { - t.Fatalf("HEAD: status = %d, want 200", w.Code) - } - if w.Body.Len() != 0 { - t.Errorf("HEAD: body length = %d, want 0", w.Body.Len()) - } -} - -func TestGenericHandler_ServesReleaseAssetCachedUnderUnhashedKey(t *testing.T) { - asset := []byte("jq binary bytes") - var available atomic.Bool - available.Store(true) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - _, _ = w.Write(asset) - })) - defer upstream.Close() - - proxy, _, store, _ := setupTestProxy(t) - store.seekable = true - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - // Cache the asset the way releases before hashed keys did. - result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), genericEcosystem, - "jqlang/jq", "jq-1.7.1", "github/jq-linux-amd64", upstream.URL+testReleaseAssetPath) - if err != nil { - t.Fatalf("priming legacy entry: %v", err) - } - if result.Reader != nil { - _ = result.Reader.Close() - } - available.Store(false) - - h := NewGenericHandler(proxy, map[string]string{"github": upstream.URL}) - w := serveGenericRequest(h, "/github"+testReleaseAssetPath) - if w.Code != http.StatusOK || w.Body.String() != string(asset) { - t.Fatalf("legacy entry: status = %d, body = %q; want 200 %q", w.Code, w.Body.String(), asset) - } - - w = serveGenericRequest(h, "/github"+testReleaseAssetPath+"?x=1") - if w.Code != http.StatusBadGateway { - t.Errorf("with query: status = %d, want 502", w.Code) - } - - repointed := NewGenericHandler(proxy, map[string]string{"github": upstream.URL + "/mirror"}) - w = serveGenericRequest(repointed, "/github"+testReleaseAssetPath) - if w.Code != http.StatusBadGateway { - t.Errorf("repointed upstream: status = %d, want 502", w.Code) - } -} - -func TestGenericHandler_ReleaseAssetNotFoundIsNotCached(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.NotFound(w, r) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewGenericHandler(proxy, map[string]string{"github": upstream.URL}) - w := serveGenericRequest(h, "/github"+testReleaseAssetPath) - if w.Code != http.StatusNotFound { - t.Fatalf("status = %d, want 404: %s", w.Code, w.Body.String()) - } -} - -func TestGenericHandler_MetadataForwardsQueryAndServesStaleOnThrottle(t *testing.T) { - const apiPath = "/repos/jqlang/jq/releases/tags/jq-1.7.1" - body := `{"tag_name":"jq-1.7.1"}` - var throttled atomic.Bool - var gotQuery string - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != apiPath { - http.NotFound(w, r) - return - } - gotQuery = r.URL.RawQuery - if throttled.Load() { - w.Header().Set("Retry-After", "60") - http.Error(w, `{"message":"API rate limit exceeded"}`, http.StatusTooManyRequests) - return - } - w.Header().Set("Content-Type", "application/vnd.github+json") - w.Header().Set("ETag", `"v1"`) - _, _ = w.Write([]byte(body)) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.CacheMetadata = true - // A tiny TTL so the second request is past freshness and has to consult - // the upstream, and the served copy is marked stale. - proxy.MetadataTTL = time.Millisecond - - h := NewGenericHandler(proxy, map[string]string{"github-api": upstream.URL}) - - req := httptest.NewRequest(http.MethodGet, "/github-api"+apiPath+"?per_page=1", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - if got := w.Body.String(); got != body { - t.Errorf("body = %q, want %q", got, body) - } - if gotQuery != "per_page=1" { - t.Errorf("upstream query = %q, want %q", gotQuery, "per_page=1") - } - if ct := w.Header().Get("Content-Type"); ct != "application/vnd.github+json" { - t.Errorf("Content-Type = %q, want upstream's", ct) - } - - // The upstream now throttles us: the cached body must be served stale - // rather than the 429 being passed through. - throttled.Store(true) - time.Sleep(5 * time.Millisecond) - w = httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("throttled: status = %d, want 200 stale: %s", w.Code, w.Body.String()) - } - if got := w.Body.String(); got != body { - t.Errorf("throttled: body = %q, want cached %q", got, body) - } - if warning := w.Header().Get("Warning"); !strings.Contains(warning, "110") { - t.Errorf("throttled: Warning = %q, want a 110 stale warning", warning) - } -} - -func TestGenericHandler_DistinctUpstreamsDoNotShareCache(t *testing.T) { - first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte("from first")) - })) - defer first.Close() - second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte("from second")) - })) - defer second.Close() - - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(first.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewGenericHandler(proxy, map[string]string{"one": first.URL, "two": second.URL}) - - w := serveGenericRequest(h, "/one"+testReleaseAssetPath) - if got := w.Body.String(); got != "from first" { - t.Fatalf("one: body = %q, want %q", got, "from first") - } - w = serveGenericRequest(h, "/two"+testReleaseAssetPath) - if got := w.Body.String(); got != "from second" { - t.Fatalf("two: body = %q, want %q (must not reuse the first upstream's cache entry)", got, "from second") - } -} - -func TestGenericHandler_UpstreamAuthIsScopedToTheConfiguredHost(t *testing.T) { - asset := []byte("private asset") - var storageAuth atomic.Value - storageAuth.Store("unset") - - // The object store the release host redirects to must never see the - // token configured for the release host. - objectStore := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - storageAuth.Store(r.Header.Get("Authorization")) - _, _ = w.Write(asset) - })) - defer objectStore.Close() - - var releaseAuth string - releaseHost := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - releaseAuth = r.Header.Get("Authorization") - if releaseAuth != "Bearer github-token" { - http.Error(w, "unauthorized", http.StatusUnauthorized) - return - } - http.Redirect(w, r, objectStore.URL+"/signed"+r.URL.Path, http.StatusFound) - })) - defer releaseHost.Close() - - proxy, _, _, _ := setupTestProxy(t) - authClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, - upstreamhttp.AuthFunc(func(url string) (string, string) { - if strings.HasPrefix(url, releaseHost.URL) { - return "Authorization", "Bearer github-token" - } - return "", "" - }))} - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(authClient), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewGenericHandler(proxy, map[string]string{"github": releaseHost.URL}) - w := serveGenericRequest(h, "/github"+testReleaseAssetPath) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - if got := w.Body.String(); got != string(asset) { - t.Errorf("body = %q, want %q", got, asset) - } - if releaseAuth != "Bearer github-token" { - t.Errorf("release host Authorization = %q, want the configured token", releaseAuth) - } - if got := storageAuth.Load(); got != "" { - t.Errorf("object store Authorization = %q, want none after the cross-host redirect", got) - } -} - -func serveGenericRequest(h *GenericHandler, target string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - return w -} - -func TestGenericHandler_UpstreamNamedHostedWhileHostingIsOff(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, "from upstream "+r.URL.Path) - })) - t.Cleanup(upstream.Close) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := NewGenericHandler(proxy, map[string]string{"hosted": upstream.URL}) - - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hosted/index.json", nil)) - if w.Code != http.StatusOK || w.Body.String() != "from upstream /index.json" { - t.Errorf("status = %d, body %q, want the upstream response", w.Code, w.Body) - } -} - -func TestGenericHandler_ServesOnlyConfiguredPaths(t *testing.T) { - var requested []string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requested = append(requested, r.URL.RequestURI()) - _, _ = io.WriteString(w, "from upstream "+r.URL.Path) - })) - t.Cleanup(upstream.Close) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewGenericHandler(proxy, map[string]string{"github-api": upstream.URL, "open": upstream.URL}). - WithPaths(map[string][]string{"github-api": {"repos/*/*/releases/**", "rate_limit"}}) - - for _, target := range []string{ - "/github-api/repos/jqlang/jq/releases/tags/jq-1.7.1", - "/github-api/rate_limit", - "/open/user/repos", - } { - if w := serveGenericRequest(h, target); w.Code != http.StatusOK { - t.Errorf("GET %s: status = %d, want 200: %s", target, w.Code, w.Body.String()) - } - } - - served := len(requested) - for _, target := range []string{ - "/github-api/user/repos", - "/github-api/repos/acme/private/contents/secret.txt", - "/github-api/repos/jqlang/jq/releases", - "/github-api/repos/jqlang/jq%3F/releases/x", - "/github-api/repos/jqlang/jq/releases/%2575ser", - "/github-api/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64", - } { - if w := serveGenericRequest(h, target); w.Code != http.StatusNotFound { - t.Errorf("GET %s: status = %d, want 404", target, w.Code) - } - } - if len(requested) != served { - t.Errorf("upstream saw %v for paths outside generic_paths", requested[served:]) - } -} diff --git a/internal/handler/git.go b/internal/handler/git.go deleted file mode 100644 index 7243d84..0000000 --- a/internal/handler/git.go +++ /dev/null @@ -1,309 +0,0 @@ -package handler - -import ( - "errors" - "net/http" - "net/url" - "strings" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/gitcache" - "github.com/git-pkgs/proxy/internal/metrics" -) - -const ( - gitEcosystem = "git" - - gitInfoRefs = "/info/refs" - gitUploadPack = "/git-upload-pack" - gitReceivePack = "/git-receive-pack" - gitRequestType = "application/x-git-upload-pack-request" - - // gitRequestTimeout replaces the server's read and write timeouts for Git - // requests once any request body has been read: a cold clone of a large - // repository keeps the client waiting before the first byte, and a large - // pack takes a while to send. - gitRequestTimeout = 2 * time.Hour - // gitWriteStallTimeout bounds each write of a Git response, so a client - // that stops reading does not hold an upload-pack process for - // gitRequestTimeout. - gitWriteStallTimeout = 2 * time.Minute -) - -// GitHandler serves Git repositories read-only over smart HTTP at -// /git/{name}/{repo}, from a local cache of the upstream.git hosts. Clients -// clone and fetch with ordinary git commands, usually through an insteadOf -// rewrite. Pushing and the dumb protocol are refused. -type GitHandler struct { - proxy *Proxy - cache *gitcache.Cache - upstreams map[string]gitUpstream - // baseURL is where clients reach the proxy, for LFS download URLs. - baseURL string - // lfsDownloads keeps upstream LFS download URLs between a batch and - // the GETs that follow it. - lfsDownloads *lfsDownloads - // writeStall is gitWriteStallTimeout, shorter in tests. - writeStall time.Duration -} - -type gitUpstream struct { - config.GitUpstream - // dir names the upstream's directory in the cache (gitcache.UpstreamDir). - dir string - // fetchWants is off when upstream.auth has credentials for the host. A - // fetch by ID there could reach commits outside the repos allowlist, - // as GitHub serves any commit in a repository's fork network. - fetchWants bool -} - -// NewGitHandler creates a Git handler serving upstream.git from cache. -func NewGitHandler(proxy *Proxy, cache *gitcache.Cache, upstream config.UpstreamConfig) *GitHandler { - h := &GitHandler{ - proxy: proxy, - cache: cache, - upstreams: make(map[string]gitUpstream, len(upstream.Git)), - writeStall: gitWriteStallTimeout, - lfsDownloads: &lfsDownloads{}, - } - for name, git := range upstream.Git { - git.URL = strings.TrimSuffix(git.URL, "/") - h.upstreams[name] = gitUpstream{ - GitUpstream: git, - dir: gitcache.UpstreamDir(name, git.URL), - fetchWants: !upstream.HasAuthWithin(git.URL), - } - } - return h -} - -// Routes returns the HTTP handler for Git requests. Mount this at /git. -func (h *GitHandler) Routes() http.Handler { - return http.HandlerFunc(h.serve) -} - -// gitRequest is a parsed /git/{name}/{repo}/{endpoint} request. -type gitRequest struct { - infoRefs bool - fetchWants bool - // key names the repository in the cache: {name}-{URL hash}/{repo}.git, - // with any .git the client gave removed from repo first, so both - // spellings share one copy. - key string - // repoURL is the upstream repository, spelled as the client spelled it. - repoURL string -} - -func (h *GitHandler) serve(w http.ResponseWriter, r *http.Request) { - if repoPath, endpoint, ok := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), gitLFSPath); ok { - h.serveLFS(w, r, repoPath, endpoint) - return - } - req, status, msg := h.parse(r) - if status != 0 { - http.Error(w, msg, status) - return - } - // An upload-pack request is read in full first, within the server's read - // timeout, so nothing waits on a client that is slow to send one. - var body *gitcache.Request - if !req.infoRefs { - var err error - if body, err = h.cache.ReadRequest(r); err != nil { - h.serveRequestError(w, req, err) - return - } - defer body.Close() - } - // The read deadline is extended too: once the body is read, an HTTP/1 - // server watches the connection, and a read timeout there would cancel - // the request. - rc := http.NewResponseController(w) - deadline := time.Now().Add(gitRequestTimeout) - if err := errors.Join(rc.SetReadDeadline(deadline), rc.SetWriteDeadline(deadline)); err != nil { - h.proxy.Logger.Warn("git: cannot extend the request deadlines", "error", err) - } - w = &stallWriter{ResponseWriter: w, rc: rc, stall: h.writeStall, deadline: deadline} - // What is left buffered once the handler returns is bounded as well. - defer func() { _ = rc.SetWriteDeadline(time.Now().Add(h.writeStall)) }() - - lease, err := h.cache.Open(r.Context(), req.key, req.repoURL, - gitcache.OpenOptions{Refresh: req.infoRefs, FetchWants: req.fetchWants}) - if err != nil { - h.serveError(w, r, err) - return - } - defer lease.Release() - if !req.infoRefs { - lease.ServeUploadPack(w, r, body) - return - } - if lease.Hit() { - metrics.RecordArtifactCacheHit(gitEcosystem) - } else { - metrics.RecordArtifactCacheMiss(gitEcosystem) - } - lease.ServeInfoRefs(w, r) -} - -func (h *GitHandler) parse(r *http.Request) (gitRequest, int, string) { - path := strings.TrimPrefix(r.URL.Path, "/") - var req gitRequest - switch { - case strings.HasSuffix(path, gitInfoRefs): - path = strings.TrimSuffix(path, gitInfoRefs) - req.infoRefs = true - switch r.URL.Query().Get("service") { - case "git-upload-pack": - case "git-receive-pack": - return req, http.StatusForbidden, "pushing is not supported" - default: - return req, http.StatusForbidden, "only the smart HTTP protocol is supported" - } - if r.Method != http.MethodGet { - return req, http.StatusMethodNotAllowed, "method not allowed" - } - case strings.HasSuffix(path, gitUploadPack): - path = strings.TrimSuffix(path, gitUploadPack) - if r.Method != http.MethodPost { - return req, http.StatusMethodNotAllowed, "method not allowed" - } - if r.Header.Get("Content-Type") != gitRequestType { - return req, http.StatusUnsupportedMediaType, "unsupported content type" - } - case strings.HasSuffix(path, gitReceivePack): - return req, http.StatusForbidden, "pushing is not supported" - default: - return req, http.StatusNotFound, "not found" - } - - repo, status, msg := h.repository(path) - if status != 0 { - return req, status, msg - } - req.key = repo.key - req.repoURL = repo.url - req.fetchWants = repo.upstream.fetchWants - return req, 0, "" -} - -// gitRepository is a repository of an upstream.git host, as a request path -// names it. -type gitRepository struct { - // name is the upstream's name and repo the path under it, without .git. - name, repo string - upstream gitUpstream - // key names the repository in the cache: {name}-{URL hash}/{repo}.git, - // with any .git the client gave removed from repo first, so both - // spellings share one copy. - key string - // url is the upstream repository, spelled as the client spelled it, and - // escapedRepo is repo escaped for a URL path. - url, escapedRepo string -} - -// repository resolves {name}/{repo} to an upstream repository, or returns -// the status and message to refuse it with. -func (h *GitHandler) repository(path string) (gitRepository, int, string) { - name, repoPath, _ := strings.Cut(path, "/") - upstream, ok := h.upstreams[name] - if !ok { - return gitRepository{}, http.StatusNotFound, "unknown upstream" - } - segments := strings.Split(repoPath, "/") - last := len(segments) - 1 - suffix := "" - if trimmed, ok := strings.CutSuffix(segments[last], ".git"); ok { - segments[last], suffix = trimmed, ".git" - } - escaped := make([]string, len(segments)) - for i, segment := range segments { - if !validGitSegment(segment, i == last) { - return gitRepository{}, http.StatusNotFound, "invalid repository path" - } - escaped[i] = url.PathEscape(segment) - } - repo := strings.Join(segments, "/") - if !upstream.AllowsRepo(repo) { - // As for a missing repository, so a prober learns nothing about - // the allowlist. - return gitRepository{}, http.StatusNotFound, "repository not found" - } - escapedRepo := strings.Join(escaped, "/") - return gitRepository{ - name: name, - repo: repo, - upstream: upstream, - key: upstream.dir + "/" + repo + ".git", - url: upstream.URL + "/" + escapedRepo + suffix, - escapedRepo: escapedRepo, - }, 0, "" -} - -// validGitSegment checks one segment of a repository path, which becomes both -// part of a directory under the cache and part of the upstream URL. Only the -// last segment may end in .git, so one repository's directory never sits -// inside another's. -func validGitSegment(segment string, last bool) bool { - if segment == "" || segment == "." || segment == ".." || (!last && strings.HasSuffix(segment, ".git")) { - return false - } - for _, r := range segment { - if r < ' ' || r == 0x7f || r == '\\' { - return false - } - } - return true -} - -// serveRequestError answers an upload-pack request whose body could not be -// read. -func (h *GitHandler) serveRequestError(w http.ResponseWriter, req gitRequest, err error) { - if errors.Is(err, gitcache.ErrRequestTooLarge) { - h.proxy.Logger.Warn("git: request body passed the limit", "repository", req.key) - http.Error(w, "request body too large", http.StatusRequestEntityTooLarge) - return - } - http.Error(w, "cannot read the request body", http.StatusBadRequest) -} - -// stallWriter bounds each write of a response by stall, where the response -// as a whole may take until deadline. -type stallWriter struct { - http.ResponseWriter - rc *http.ResponseController - stall time.Duration - deadline time.Time -} - -func (w *stallWriter) Write(p []byte) (int, error) { - _ = w.rc.SetWriteDeadline(time.Now().Add(w.stall)) - n, err := w.ResponseWriter.Write(p) - _ = w.rc.SetWriteDeadline(w.deadline) - return n, err -} - -func (w *stallWriter) Unwrap() http.ResponseWriter { - return w.ResponseWriter -} - -// serveError answers a request whose repository could not be opened. The -// cache has already logged why. -func (h *GitHandler) serveError(w http.ResponseWriter, r *http.Request, err error) { - if r.Context().Err() != nil { - return - } - switch { - case errors.Is(err, gitcache.ErrNotFound): - http.Error(w, "repository not found", http.StatusNotFound) - case errors.Is(err, gitcache.ErrRefused): - http.Error(w, "upstream refused the request", http.StatusBadGateway) - case errors.Is(err, gitcache.ErrTooLarge): - http.Error(w, "repository is larger than git.max_repo_size or git.max_size", http.StatusInsufficientStorage) - case errors.Is(err, gitcache.ErrCacheFull): - http.Error(w, "git cache is full", http.StatusInsufficientStorage) - default: - http.Error(w, "upstream unavailable", http.StatusBadGateway) - } -} diff --git a/internal/handler/git_lfs.go b/internal/handler/git_lfs.go deleted file mode 100644 index d4f6fe6..0000000 --- a/internal/handler/git_lfs.go +++ /dev/null @@ -1,559 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "slices" - "strconv" - "strings" - "sync" - "time" -) - -const ( - // gitLFSPath separates a repository from the Git LFS API below it, as - // git-lfs derives the API URL from a remote: {remote}.git/info/lfs. - gitLFSPath = "/info/lfs/" - gitLFSMediaType = "application/vnd.git-lfs+json" - gitLFSTransfer = "basic" - gitLFSHashAlgo = "sha256" - // gitLFSObjectMediaType is what objects are served as. - gitLFSObjectMediaType = "application/octet-stream" - - // gitLFSMaxBatchRequest bounds a client's batch request. git-lfs sends - // 100 objects per batch by default, a few kilobytes. - gitLFSMaxBatchRequest = 1 << 20 - // gitLFSMaxBatchResponse bounds an upstream batch response, which holds - // an href and headers per object. - gitLFSMaxBatchResponse = 16 << 20 - - gitLFSOIDLength = 64 -) - -// lfsBatchRequest is a Git LFS batch API request. -type lfsBatchRequest struct { - Operation string `json:"operation"` - Transfers []string `json:"transfers,omitempty"` - Ref json.RawMessage `json:"ref,omitempty"` - Objects []lfsObject `json:"objects"` - HashAlgo string `json:"hash_algo,omitempty"` -} - -type lfsObject struct { - OID string `json:"oid"` - Size int64 `json:"size"` -} - -// lfsBatchResponse is a Git LFS batch API response. -type lfsBatchResponse struct { - Transfer string `json:"transfer,omitempty"` - Objects []lfsObjectResponse `json:"objects"` - HashAlgo string `json:"hash_algo,omitempty"` -} - -type lfsObjectResponse struct { - OID string `json:"oid"` - Size int64 `json:"size"` - Authenticated bool `json:"authenticated,omitempty"` - Actions map[string]lfsAction `json:"actions,omitempty"` - Error *lfsObjectError `json:"error,omitempty"` -} - -type lfsAction struct { - Href string `json:"href"` - Header map[string]string `json:"header,omitempty"` - ExpiresIn int64 `json:"expires_in,omitempty"` - ExpiresAt string `json:"expires_at,omitempty"` -} - -type lfsObjectError struct { - Code int `json:"code"` - Message string `json:"message"` -} - -// lfsUpstreamError is a batch request the upstream answered with an error -// status. -type lfsUpstreamError struct { - status int -} - -func (e *lfsUpstreamError) Error() string { - return fmt.Sprintf("LFS batch API answered %d", e.status) -} - -// WithBaseURL sets the URL clients reach the proxy at, which LFS batch -// responses send them back to for each object. -func (h *GitHandler) WithBaseURL(baseURL string) *GitHandler { - h.baseURL = strings.TrimSuffix(baseURL, "/") - return h -} - -// serveLFS answers a request to the Git LFS API of a repository: batch -// downloads, and the objects the batch responses point at. Objects are -// stored in the artifact cache, checked against their sha256 object ID. -// Uploads and locking are refused. -func (h *GitHandler) serveLFS(w http.ResponseWriter, r *http.Request, repoPath, endpoint string) { - repo, status, msg := h.repository(repoPath) - if status != 0 { - lfsError(w, status, msg) - return - } - if endpoint == "objects/batch" { - if r.Method != http.MethodPost { - lfsError(w, http.StatusMethodNotAllowed, "method not allowed") - return - } - h.serveLFSBatch(w, r, repo) - return - } - oid, sizeText, ok := strings.Cut(strings.TrimPrefix(endpoint, "objects/"), "/") - size, err := strconv.ParseInt(sizeText, 10, 64) - if !strings.HasPrefix(endpoint, "objects/") || !ok || err != nil || size < 0 || !validLFSOID(oid) { - lfsError(w, http.StatusNotFound, "not found") - return - } - if r.Method != http.MethodGet && r.Method != http.MethodHead { - lfsError(w, http.StatusMethodNotAllowed, "method not allowed") - return - } - h.serveLFSObject(w, r, repo, lfsObject{OID: oid, Size: size}) -} - -// serveLFSBatch forwards a download batch upstream and points every object -// the upstream offers at this proxy. The upstream answers for the -// repository, so a client only gets objects it could get from there. -func (h *GitHandler) serveLFSBatch(w http.ResponseWriter, r *http.Request, repo gitRepository) { - var req lfsBatchRequest - if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, gitLFSMaxBatchRequest)).Decode(&req); err != nil { - var tooLarge *http.MaxBytesError - if errors.As(err, &tooLarge) { - lfsError(w, http.StatusRequestEntityTooLarge, "batch request too large") - return - } - lfsError(w, http.StatusBadRequest, "invalid batch request") - return - } - switch { - case req.Operation != "download": - lfsError(w, http.StatusForbidden, "pushing is not supported") - return - case req.HashAlgo != "" && req.HashAlgo != gitLFSHashAlgo: - lfsError(w, http.StatusConflict, "only sha256 objects are supported") - return - case len(req.Transfers) > 0 && !slices.Contains(req.Transfers, gitLFSTransfer): - lfsError(w, http.StatusUnprocessableEntity, "only the basic transfer is supported") - return - } - requested := make(map[string]int64, len(req.Objects)) - for _, object := range req.Objects { - if !validLFSOID(object.OID) || object.Size < 0 { - lfsError(w, http.StatusUnprocessableEntity, "invalid object") - return - } - requested[object.OID] = object.Size - } - - upstream, err := h.lfsBatch(r.Context(), repo, req.Ref, req.Objects) - if err != nil { - if resp, ok := h.lfsBatchFromCache(r, repo, req.Objects, err); ok { - writeLFSBatch(w, resp) - return - } - h.serveLFSUpstreamError(w, r, repo, err) - return - } - resp := newLFSBatchResponse() - for _, object := range upstream.Objects { - // Only objects the client asked for, at the size it gave, so a - // response can't point the client at anything else. - size, ok := requested[object.OID] - if !ok || size != object.Size { - continue - } - out := lfsObjectResponse{OID: object.OID, Size: object.Size, Error: object.Error} - if download, ok := object.Actions["download"]; ok && object.Error == nil && download.Href != "" { - out.Authenticated = true - out.Actions = map[string]lfsAction{"download": {Href: h.lfsObjectURL(repo, object.OID, object.Size)}} - // Kept so the client's GET for the object needs no second - // batch request. - h.lfsDownloads.put(repo.key, lfsObject{OID: object.OID, Size: object.Size}, download, time.Now()) - } - resp.Objects = append(resp.Objects, out) - } - writeLFSBatch(w, resp) -} - -// lfsBatchFromCache answers a batch the upstream couldn't, because it was -// unreachable or failing, from the objects cached for repo. This is what a -// GET of each object would serve anyway. A refusal by the upstream is not -// overridden, and a batch none of whose objects are cached is left to fail. -func (h *GitHandler) lfsBatchFromCache(r *http.Request, repo gitRepository, objects []lfsObject, err error) (lfsBatchResponse, bool) { - var refused *lfsUpstreamError - if r.Context().Err() != nil || (errors.As(err, &refused) && refused.status < http.StatusInternalServerError) { - return lfsBatchResponse{}, false - } - resp := newLFSBatchResponse() - seen := make(map[string]bool, len(objects)) - cached := 0 - for _, object := range objects { - if seen[object.OID] { - continue - } - seen[object.OID] = true - out := lfsObjectResponse{OID: object.OID, Size: object.Size} - if ok, _ := h.proxy.HasCachedArtifact(r.Context(), gitEcosystem, repo.lfsName(), object.OID, object.OID); ok { - out.Authenticated = true - out.Actions = map[string]lfsAction{"download": {Href: h.lfsObjectURL(repo, object.OID, object.Size)}} - cached++ - } else { - out.Error = &lfsObjectError{Code: http.StatusBadGateway, Message: "upstream unavailable and object not cached"} - } - resp.Objects = append(resp.Objects, out) - } - if cached == 0 { - return lfsBatchResponse{}, false - } - h.proxy.Logger.Warn("git: LFS batch request failed, answering from the cache", - "repository", repo.key, "cached", cached, "objects", len(resp.Objects), "error", err) - return resp, true -} - -func newLFSBatchResponse() lfsBatchResponse { - return lfsBatchResponse{Transfer: gitLFSTransfer, Objects: []lfsObjectResponse{}, HashAlgo: gitLFSHashAlgo} -} - -func writeLFSBatch(w http.ResponseWriter, resp lfsBatchResponse) { - w.Header().Set("Content-Type", gitLFSMediaType) - _ = json.NewEncoder(w).Encode(resp) -} - -// serveLFSObject serves an object from the artifact cache. On a miss it -// takes the download URL from the client's batch, or asks the upstream batch -// API for a fresh one, and caches the object from there, or from a copy -// already stored under another repository, once its content matches its -// object ID. -func (h *GitHandler) serveLFSObject(w http.ResponseWriter, r *http.Request, repo gitRepository, object lfsObject) { - cached, err := h.proxy.GetCachedArtifact(r.Context(), gitEcosystem, repo.lfsName(), object.OID, object.OID) - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to read cached LFS object") - return - } - if cached == nil { - var ok bool - if cached, ok = h.fetchLFSObject(w, r, repo, object); !ok { - return - } - } - serveGenericArtifact(w, r, cached) -} - -// fetchLFSObject caches an object the cache doesn't have. When it can't, or -// the request is a HEAD, it answers the request and returns false. -func (h *GitHandler) fetchLFSObject(w http.ResponseWriter, r *http.Request, repo gitRepository, object lfsObject) (*CacheResult, bool) { - download, kept := h.lfsDownloads.get(repo.key, object, time.Now()) - if !kept { - var ok bool - if download, ok = h.lfsFreshDownload(w, r, repo, object); !ok { - return nil, false - } - } - if r.Method == http.MethodHead { - // The upstream has the object, and a HEAD needs no more than that: - // fetching it is left to a GET. - w.Header().Set("Content-Type", gitLFSObjectMediaType) - w.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10)) - return nil, false - } - h.lfsDownloads.drop(repo.key, object.OID) - cached, err := h.cacheLFSObject(r.Context(), repo, object, download) - if err != nil && kept && retryLFSDownload(r.Context(), err) { - // The kept URL may have expired early or been revoked. - var ok bool - if download, ok = h.lfsFreshDownload(w, r, repo, object); !ok { - return nil, false - } - cached, err = h.cacheLFSObject(r.Context(), repo, object, download) - } - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch LFS object") - return nil, false - } - return cached, true -} - -// lfsFreshDownload asks the upstream batch API for object's download. When -// there is none it answers the request and returns false. -func (h *GitHandler) lfsFreshDownload(w http.ResponseWriter, r *http.Request, repo gitRepository, object lfsObject) (lfsAction, bool) { - upstream, err := h.lfsBatch(r.Context(), repo, nil, []lfsObject{object}) - if err != nil { - h.serveLFSUpstreamError(w, r, repo, err) - return lfsAction{}, false - } - download, status, msg := lfsDownload(upstream, object) - if status != 0 { - lfsError(w, status, msg) - return lfsAction{}, false - } - return download, true -} - -// cacheLFSObject caches object, which the upstream offers as download. A -// copy already stored under another name, such as by a fork of the -// repository, saves the download. -func (h *GitHandler) cacheLFSObject(ctx context.Context, repo gitRepository, object lfsObject, download lfsAction) (*CacheResult, error) { - header := make(http.Header, len(download.Header)) - for key, value := range download.Header { - header.Set(key, value) - } - // Download URLs are often signed, granting access to the object until - // they expire, so only the URL without its query is logged and - // recorded. The object must have the size the client asked for. - ctx, recorded := withSignedDownload(ctx, download.Href, object.Size) - if !h.proxy.StreamArtifacts { - stored, err := h.proxy.DB.FindStoredContent(object.OID) - if err != nil { - h.proxy.Logger.Warn("git: looking up a stored copy of an LFS object failed", "oid", object.OID, "error", err) - } - if stored != nil && stored.Size == object.Size { - cached, err := h.proxy.cacheStoredCopy(ctx, gitEcosystem, repo.lfsName(), object.OID, object.OID, - recorded, object.OID, stored) - if err == nil { - return cached, nil - } - h.proxy.Logger.Warn("git: copying a stored LFS object failed, downloading it", "oid", object.OID, "error", err) - } - } - return h.proxy.getOrFetchArtifactFromURL(ctx, gitEcosystem, repo.lfsName(), object.OID, object.OID, - recorded, header, object.OID, "") -} - -// retryLFSDownload reports whether a download from a kept URL failed in a way -// a fresh URL might not. -func retryLFSDownload(ctx context.Context, err error) bool { - return ctx.Err() == nil && - !errors.Is(err, ErrArtifactDigestMismatch) && - !errors.Is(err, errDownloadSize) && - !errors.Is(err, ErrVersionDenied) && - !errors.Is(err, ErrArtifactBlocked) -} - -// lfsDownload returns the download action the upstream gave for object, or -// the status and message to answer with when it gave none. -func lfsDownload(resp *lfsBatchResponse, object lfsObject) (lfsAction, int, string) { - for _, got := range resp.Objects { - if got.OID != object.OID { - continue - } - if got.Error != nil { - if got.Error.Code == http.StatusNotFound || got.Error.Code == http.StatusGone { - return lfsAction{}, http.StatusNotFound, "object not found" - } - return lfsAction{}, http.StatusBadGateway, "upstream refused the object" - } - download, ok := got.Actions["download"] - if !ok || !validLFSHref(download.Href) { - return lfsAction{}, http.StatusBadGateway, "upstream gave no download for the object" - } - return download, 0, "" - } - return lfsAction{}, http.StatusNotFound, "object not found" -} - -// validLFSHref reports whether href is an absolute HTTP(S) URL. -func validLFSHref(href string) bool { - u, err := url.Parse(href) - return err == nil && (u.Scheme == "https" || u.Scheme == "http") && u.Host != "" -} - -// lfsBatch asks the upstream's LFS batch API for download actions for -// objects. upstream.auth applies to it as to the repository. -func (h *GitHandler) lfsBatch(ctx context.Context, repo gitRepository, ref json.RawMessage, objects []lfsObject) (*lfsBatchResponse, error) { - body, err := json.Marshal(lfsBatchRequest{ - Operation: "download", - Transfers: []string{gitLFSTransfer}, - Ref: ref, - Objects: objects, - HashAlgo: gitLFSHashAlgo, - }) - if err != nil { - return nil, err - } - batchURL := strings.TrimSuffix(repo.url, ".git") + ".git" + gitLFSPath + "objects/batch" - req, err := http.NewRequestWithContext(ctx, http.MethodPost, batchURL, bytes.NewReader(body)) - if err != nil { - return nil, err - } - req.Header.Set("Accept", gitLFSMediaType) - req.Header.Set("Content-Type", gitLFSMediaType) - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - return nil, &lfsUpstreamError{status: resp.StatusCode} - } - var batch lfsBatchResponse - if err := json.NewDecoder(io.LimitReader(resp.Body, gitLFSMaxBatchResponse)).Decode(&batch); err != nil { - return nil, fmt.Errorf("reading LFS batch response: %w", err) - } - if batch.Transfer != "" && batch.Transfer != gitLFSTransfer { - return nil, fmt.Errorf("LFS batch response uses the %q transfer", batch.Transfer) - } - return &batch, nil -} - -// serveLFSUpstreamError answers a request whose upstream batch request -// failed. -func (h *GitHandler) serveLFSUpstreamError(w http.ResponseWriter, r *http.Request, repo gitRepository, err error) { - if r.Context().Err() != nil { - return - } - h.proxy.Logger.Warn("git: LFS batch request failed", "repository", repo.key, "error", err) - var upstream *lfsUpstreamError - switch { - case errors.As(err, &upstream) && upstream.status == http.StatusNotFound: - lfsError(w, http.StatusNotFound, "repository not found") - case errors.As(err, &upstream) && upstream.status < http.StatusInternalServerError: - lfsError(w, http.StatusBadGateway, "upstream refused the request") - default: - lfsError(w, http.StatusBadGateway, "upstream unavailable") - } -} - -// lfsName names repo's LFS objects in the artifact cache. Objects are cached -// per repository: having one cached for another repository says nothing -// about whether this one has it. -func (repo gitRepository) lfsName() string { - return repo.name + "/" + repo.repo -} - -// lfsObjectURL is where this proxy serves an object of repo. -func (h *GitHandler) lfsObjectURL(repo gitRepository, oid string, size int64) string { - return fmt.Sprintf("%s/git/%s/%s.git%sobjects/%s/%d", - h.baseURL, url.PathEscape(repo.name), repo.escapedRepo, gitLFSPath, oid, size) -} - -// validLFSOID reports whether oid is a sha256 object ID in lowercase hex. -func validLFSOID(oid string) bool { - if len(oid) != gitLFSOIDLength { - return false - } - for _, c := range oid { - if (c < '0' || c > '9') && (c < 'a' || c > 'f') { - return false - } - } - return true -} - -// lfsError answers with a Git LFS error document, which git-lfs shows. -func lfsError(w http.ResponseWriter, status int, msg string) { - w.Header().Set("Content-Type", gitLFSMediaType) - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(map[string]string{"message": msg}) -} - -const ( - // gitLFSKeptDownloads bounds how many download URLs from batch responses - // are kept for the GETs that follow them. - gitLFSKeptDownloads = 4096 - // gitLFSKeptTTL is how long a download URL is kept when the upstream - // gives no expiry, and gitLFSKeptMaxTTL how long at most. - gitLFSKeptTTL = 5 * time.Minute - gitLFSKeptMaxTTL = time.Hour - // gitLFSExpiryMargin is how long before its expiry a URL is no longer - // used, so a download started with it can finish. - gitLFSExpiryMargin = 30 * time.Second -) - -// lfsDownloads keeps the download actions of recent batch responses, so the -// GET for an object a client just got in a batch needs no second batch -// request. They stay in memory, never reach a client and expire with the -// URLs. -type lfsDownloads struct { - mu sync.Mutex - entries map[lfsDownloadKey]lfsKeptDownload -} - -type lfsDownloadKey struct { - repo, oid string -} - -type lfsKeptDownload struct { - size int64 - action lfsAction - expires time.Time -} - -// put keeps action for object of repo until shortly before it expires. -func (d *lfsDownloads) put(repo string, object lfsObject, action lfsAction, now time.Time) { - if !validLFSHref(action.Href) { - return - } - expires := now.Add(gitLFSKeptTTL) - if action.ExpiresIn > 0 || action.ExpiresAt != "" { - expires = now.Add(gitLFSKeptMaxTTL) - } - if action.ExpiresIn > 0 { - expires = minTime(expires, now.Add(time.Duration(action.ExpiresIn)*time.Second)) - } - if at, err := time.Parse(time.RFC3339, action.ExpiresAt); err == nil { - expires = minTime(expires, at) - } - expires = expires.Add(-gitLFSExpiryMargin) - if !expires.After(now) { - return - } - d.mu.Lock() - defer d.mu.Unlock() - if d.entries == nil { - d.entries = make(map[lfsDownloadKey]lfsKeptDownload) - } - if len(d.entries) >= gitLFSKeptDownloads { - for key, kept := range d.entries { - if !kept.expires.After(now) { - delete(d.entries, key) - } - } - } - if len(d.entries) >= gitLFSKeptDownloads { - // Still full: make room by dropping any one. - for key := range d.entries { - delete(d.entries, key) - break - } - } - d.entries[lfsDownloadKey{repo, object.OID}] = lfsKeptDownload{size: object.Size, action: action, expires: expires} -} - -// get returns the action kept for object of repo, if it hasn't expired. -func (d *lfsDownloads) get(repo string, object lfsObject, now time.Time) (lfsAction, bool) { - d.mu.Lock() - defer d.mu.Unlock() - kept, ok := d.entries[lfsDownloadKey{repo, object.OID}] - if !ok || kept.size != object.Size || !kept.expires.After(now) { - return lfsAction{}, false - } - return kept.action, true -} - -// drop forgets the action kept for oid of repo, once a GET has used it. -func (d *lfsDownloads) drop(repo, oid string) { - d.mu.Lock() - defer d.mu.Unlock() - delete(d.entries, lfsDownloadKey{repo, oid}) -} - -func minTime(a, b time.Time) time.Time { - if b.Before(a) { - return b - } - return a -} diff --git a/internal/handler/git_lfs_test.go b/internal/handler/git_lfs_test.go deleted file mode 100644 index 37bc6aa..0000000 --- a/internal/handler/git_lfs_test.go +++ /dev/null @@ -1,483 +0,0 @@ -package handler - -import ( - "bytes" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/registries/fetch" -) - -const lfsTestContent = "large file content" - -// lfsSignature is the signature in the upstream's download URLs, which must -// not reach logs or the database. -const lfsSignature = "s3cr3t-signature" - -// lfsUpstream is a Git host with an LFS batch API whose download hrefs point -// at a separate object store, as GitHub's do. The store only serves objects -// to requests carrying the header and signed URL the batch response gave. -type lfsUpstream struct { - git, store *httptest.Server - oid string - content atomic.Value // string the store serves for oid - batches atomic.Int32 - downloads atomic.Int32 - // batchStatus, when set, is the status the batch API fails with. - batchStatus atomic.Int32 - // signature is the signature the batch API hands out and the store - // accepts; changing it expires the URLs handed out before. - signature atomic.Value - // lastBatch is the last batch request the upstream got. - lastBatch atomic.Value - // logs is what the proxy logged. - logs bytes.Buffer -} - -func newLFSUpstream(t *testing.T) *lfsUpstream { - t.Helper() - sum := sha256.Sum256([]byte(lfsTestContent)) - u := &lfsUpstream{oid: hex.EncodeToString(sum[:])} - u.content.Store(lfsTestContent) - u.signature.Store(lfsSignature) - u.store = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/objects/"+u.oid || r.Header.Get("X-Signed") != "token" || - r.URL.Query().Get("Signature") != u.signature.Load().(string) { - http.Error(w, "forbidden", http.StatusForbidden) - return - } - u.downloads.Add(1) - _, _ = io.WriteString(w, u.content.Load().(string)) - })) - u.git = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // acme/widget has the object, as does its fork; stranger/widget - // exists but doesn't have it. - repo, _, _ := strings.Cut(r.URL.Path, ".git/info/lfs/objects/batch") - hasObject, exists := map[string]bool{"/acme/widget": true, "/fork/widget": true, "/stranger/widget": false}[repo] - if r.Method != http.MethodPost || !exists { - http.NotFound(w, r) - return - } - u.batches.Add(1) - if status := u.batchStatus.Load(); status != 0 { - http.Error(w, "failing", int(status)) - return - } - var req lfsBatchRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - u.lastBatch.Store(req) - resp := lfsBatchResponse{Transfer: "basic"} - for _, object := range req.Objects { - if object.OID != u.oid || !hasObject { - resp.Objects = append(resp.Objects, lfsObjectResponse{OID: object.OID, Size: object.Size, - Error: &lfsObjectError{Code: http.StatusNotFound, Message: "Object does not exist"}}) - continue - } - resp.Objects = append(resp.Objects, lfsObjectResponse{OID: object.OID, Size: object.Size, - Actions: map[string]lfsAction{"download": { - Href: u.store.URL + "/objects/" + u.oid + "?Expires=1&Signature=" + u.signature.Load().(string), - Header: map[string]string{"X-Signed": "token"}, - }}}) - } - // An object nobody asked for is not passed on. - resp.Objects = append(resp.Objects, lfsObjectResponse{OID: strings.Repeat("e", 64), Size: 1, - Actions: map[string]lfsAction{"download": {Href: u.store.URL + "/other"}}}) - w.Header().Set("Content-Type", gitLFSMediaType) - _ = json.NewEncoder(w).Encode(resp) - })) - t.Cleanup(u.git.Close) - t.Cleanup(u.store.Close) - return u -} - -func newLFSHandler(t *testing.T, u *lfsUpstream, repos ...string) *GitHandler { - t.Helper() - proxy, _, store, _ := setupTestProxy(t) - store.seekable = true - proxy.Logger = slog.New(slog.NewTextHandler(&u.logs, nil)) - proxy.HTTPClient = u.git.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(u.store.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - return NewGitHandler(proxy, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ - "up": {URL: u.git.URL, Repos: repos}, - }}).WithBaseURL("http://proxy.test/") -} - -func lfsRequest(h *GitHandler, method, target, body string) *httptest.ResponseRecorder { - r := httptest.NewRequest(method, target, strings.NewReader(body)) - r.Header.Set("Content-Type", gitLFSMediaType) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, r) - return w -} - -func lfsBatchBody(operation string, objects ...lfsObject) string { - body, _ := json.Marshal(lfsBatchRequest{Operation: operation, Transfers: []string{"basic"}, - Ref: json.RawMessage(`{"name":"refs/heads/main"}`), Objects: objects}) - return string(body) -} - -func TestGitLFSBatchPointsObjectsAtTheProxy(t *testing.T) { - u := newLFSUpstream(t) - h := newLFSHandler(t, u) - missing := strings.Repeat("0", 64) - size := int64(len(lfsTestContent)) - - w := lfsRequest(h, http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", - lfsBatchBody("download", lfsObject{OID: u.oid, Size: size}, lfsObject{OID: missing, Size: 3})) - if w.Code != http.StatusOK || w.Header().Get("Content-Type") != gitLFSMediaType { - t.Fatalf("batch: status %d, type %q, body %s", w.Code, w.Header().Get("Content-Type"), w.Body) - } - var resp lfsBatchResponse - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } - if len(resp.Objects) != 2 { - t.Fatalf("objects = %+v, want the two requested", resp.Objects) - } - got, notFound := resp.Objects[0], resp.Objects[1] - want := fmt.Sprintf("http://proxy.test/git/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size) - if download := got.Actions["download"]; download.Href != want || len(download.Header) != 0 || !got.Authenticated { - t.Errorf("download = %+v, authenticated %v; want %s without the upstream's headers", download, got.Authenticated, want) - } - if notFound.Error == nil || notFound.Error.Code != http.StatusNotFound || notFound.Actions != nil { - t.Errorf("missing object = %+v, want the upstream's 404", notFound) - } - if req, _ := u.lastBatch.Load().(lfsBatchRequest); string(req.Ref) != `{"name":"refs/heads/main"}` { - t.Errorf("upstream got ref %s, want the client's", req.Ref) - } -} - -func TestGitLFSObjectIsCached(t *testing.T) { - u := newLFSUpstream(t) - h := newLFSHandler(t, u) - path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) - - w := lfsRequest(h, http.MethodGet, path, "") - if w.Code != http.StatusOK || w.Body.String() != lfsTestContent { - t.Fatalf("first GET: status %d, body %q", w.Code, w.Body) - } - if u.batches.Load() != 1 || u.downloads.Load() != 1 { - t.Errorf("batches %d, downloads %d; want one of each", u.batches.Load(), u.downloads.Load()) - } - - u.git.Close() - u.store.Close() - w = lfsRequest(h, http.MethodGet, path, "") - if w.Code != http.StatusOK || w.Body.String() != lfsTestContent { - t.Errorf("GET with the upstream down: status %d, body %q; want it from the cache", w.Code, w.Body) - } -} - -func TestGitLFSObjectRefusesWrongContent(t *testing.T) { - u := newLFSUpstream(t) - u.content.Store("tampered content!!") - h := newLFSHandler(t, u) - path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) - - for range 2 { - if w := lfsRequest(h, http.MethodGet, path, ""); w.Code == http.StatusOK && w.Body.String() == "tampered content!!" { - t.Fatal("served content that does not match its object ID") - } - } - if u.downloads.Load() != 2 { - t.Errorf("downloads = %d, want 2: the mismatching object must not be cached", u.downloads.Load()) - } -} - -func TestGitLFSRefusals(t *testing.T) { - u := newLFSUpstream(t) - h := newLFSHandler(t, u, "acme/*") - object := lfsObject{OID: u.oid, Size: int64(len(lfsTestContent))} - tests := []struct { - name, method, target, body string - status int - }{ - {"upload", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", lfsBatchBody("upload", object), http.StatusForbidden}, - {"sha512 objects", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", - `{"operation":"download","objects":[],"hash_algo":"sha512"}`, http.StatusConflict}, - {"other transfers", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", - `{"operation":"download","transfers":["ssh"],"objects":[]}`, http.StatusUnprocessableEntity}, - {"invalid oid", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", - lfsBatchBody("download", lfsObject{OID: "../x", Size: 1}), http.StatusUnprocessableEntity}, - {"batch GET", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/batch", "", http.StatusMethodNotAllowed}, - {"repository outside repos", http.MethodPost, "/up/other/widget.git/info/lfs/objects/batch", lfsBatchBody("download", object), http.StatusNotFound}, - {"unknown repository", http.MethodPost, "/up/acme/missing.git/info/lfs/objects/batch", lfsBatchBody("download", object), http.StatusNotFound}, - {"object without size", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/" + u.oid, "", http.StatusNotFound}, - {"uppercase oid", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/" + strings.ToUpper(u.oid) + "/1", "", http.StatusNotFound}, - {"object PUT", http.MethodPut, "/up/acme/widget.git/info/lfs/objects/" + u.oid + "/1", "", http.StatusMethodNotAllowed}, - {"locks", http.MethodPost, "/up/acme/widget.git/info/lfs/locks/verify", "{}", http.StatusNotFound}, - {"missing object", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/" + strings.Repeat("0", 64) + "/3", "", http.StatusNotFound}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - w := lfsRequest(h, tt.method, tt.target, tt.body) - if w.Code != tt.status || w.Header().Get("Content-Type") != gitLFSMediaType { - t.Errorf("status %d, type %q, body %s; want %d", w.Code, w.Header().Get("Content-Type"), w.Body, tt.status) - } - }) - } - if u.downloads.Load() != 0 { - t.Errorf("downloads = %d, want none", u.downloads.Load()) - } -} - -func TestGitLFSKeepsSignedURLsOutOfLogsAndDatabase(t *testing.T) { - u := newLFSUpstream(t) - h := newLFSHandler(t, u) - path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) - - if w := lfsRequest(h, http.MethodGet, path, ""); w.Code != http.StatusOK { - t.Fatalf("GET: status %d, body %s", w.Code, w.Body) - } - _, versionPURL, err := packagePURLStrings(gitEcosystem, "up/acme/widget", u.oid) - if err != nil { - t.Fatal(err) - } - record, err := h.proxy.DB.GetArtifact(versionPURL, u.oid) - if err != nil || record == nil { - t.Fatalf("artifact record: %v, %v", record, err) - } - if want := u.store.URL + "/objects/" + u.oid; record.UpstreamURL != want { - t.Errorf("recorded upstream URL %q, want %q", record.UpstreamURL, want) - } - - // A failed download is logged without the signature too, although the - // connection error names the URL. - other := newLFSUpstream(t) - other.store.Close() - h = newLFSHandler(t, other) - if w := lfsRequest(h, http.MethodGet, path, ""); w.Code == http.StatusOK { - t.Fatalf("GET from an unreachable store: status %d", w.Code) - } - if !strings.Contains(other.logs.String(), other.store.URL+"/objects/"+other.oid) { - t.Fatalf("logs don't name the failed download:\n%s", other.logs.String()) - } - for _, logs := range []string{u.logs.String(), other.logs.String()} { - if strings.Contains(logs, lfsSignature) { - t.Errorf("logs contain the download URL's signature:\n%s", logs) - } - } -} - -func TestGitLFSBatchFromCacheWhenUpstreamFails(t *testing.T) { - u := newLFSUpstream(t) - h := newLFSHandler(t, u) - size := int64(len(lfsTestContent)) - missing := strings.Repeat("0", 64) - batch := lfsBatchBody("download", lfsObject{OID: u.oid, Size: size}, lfsObject{OID: missing, Size: 3}) - const batchPath = "/up/acme/widget.git/info/lfs/objects/batch" - - // Nothing cached yet: a failing upstream fails the batch. - u.batchStatus.Store(http.StatusServiceUnavailable) - if w := lfsRequest(h, http.MethodPost, batchPath, batch); w.Code != http.StatusBadGateway { - t.Fatalf("batch with nothing cached: status %d, body %s; want 502", w.Code, w.Body) - } - u.batchStatus.Store(0) - if w := lfsRequest(h, http.MethodGet, fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size), ""); w.Code != http.StatusOK { - t.Fatalf("GET: status %d, body %s", w.Code, w.Body) - } - - // A refusal is the upstream's answer, not an outage. - u.batchStatus.Store(http.StatusForbidden) - if w := lfsRequest(h, http.MethodPost, batchPath, batch); w.Code != http.StatusBadGateway { - t.Errorf("batch the upstream refused: status %d, body %s; want 502", w.Code, w.Body) - } - - u.git.Close() - w := lfsRequest(h, http.MethodPost, batchPath, batch) - if w.Code != http.StatusOK { - t.Fatalf("batch with the upstream down: status %d, body %s", w.Code, w.Body) - } - var resp lfsBatchResponse - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } - if len(resp.Objects) != 2 { - t.Fatalf("objects = %+v, want the two requested", resp.Objects) - } - want := fmt.Sprintf("http://proxy.test/git/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size) - if got := resp.Objects[0]; got.Actions["download"].Href != want || !got.Authenticated || got.Error != nil { - t.Errorf("cached object = %+v, want a download from %s", got, want) - } - if got := resp.Objects[1]; got.Actions != nil || got.Error == nil { - t.Errorf("uncached object = %+v, want an error", got) - } -} - -func TestGitLFSHeadDoesNotDownload(t *testing.T) { - u := newLFSUpstream(t) - h := newLFSHandler(t, u) - path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) - - w := lfsRequest(h, http.MethodHead, path, "") - if w.Code != http.StatusOK || w.Header().Get("Content-Length") != fmt.Sprint(len(lfsTestContent)) { - t.Errorf("HEAD: status %d, Content-Length %q", w.Code, w.Header().Get("Content-Length")) - } - if u.downloads.Load() != 0 { - t.Errorf("downloads = %d, want none for a HEAD", u.downloads.Load()) - } - missing := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/3", strings.Repeat("0", 64)) - if w := lfsRequest(h, http.MethodHead, missing, ""); w.Code != http.StatusNotFound { - t.Errorf("HEAD of a missing object: status %d, want 404", w.Code) - } -} - -func TestGitLFSObjectAfterBatchNeedsNoSecondBatch(t *testing.T) { - u := newLFSUpstream(t) - h := newLFSHandler(t, u) - size := int64(len(lfsTestContent)) - path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size) - batch := lfsBatchBody("download", lfsObject{OID: u.oid, Size: size}) - - if w := lfsRequest(h, http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", batch); w.Code != http.StatusOK { - t.Fatalf("batch: status %d, body %s", w.Code, w.Body) - } - if w := lfsRequest(h, http.MethodGet, path, ""); w.Code != http.StatusOK || w.Body.String() != lfsTestContent { - t.Fatalf("GET: status %d, body %q", w.Code, w.Body) - } - if u.batches.Load() != 1 || u.downloads.Load() != 1 { - t.Errorf("batches %d, downloads %d; want the client's batch and one download", u.batches.Load(), u.downloads.Load()) - } - - // A URL that stopped working before its time is replaced by a fresh one. - other := newLFSUpstream(t) - h = newLFSHandler(t, other) - if w := lfsRequest(h, http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", batch); w.Code != http.StatusOK { - t.Fatalf("batch: status %d, body %s", w.Code, w.Body) - } - other.signature.Store("rotated") - if w := lfsRequest(h, http.MethodGet, path, ""); w.Code != http.StatusOK || w.Body.String() != lfsTestContent { - t.Fatalf("GET with a revoked URL: status %d, body %q", w.Code, w.Body) - } - if other.batches.Load() != 2 { - t.Errorf("batches = %d, want a second one for a fresh URL", other.batches.Load()) - } -} - -func TestGitLFSKeptDownloadsExpire(t *testing.T) { - now := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC) - object := lfsObject{OID: strings.Repeat("a", 64), Size: 3} - href := "https://objects.example/a?sig=1" - tests := []struct { - name string - action lfsAction - at time.Duration - kept bool - }{ - {"no expiry", lfsAction{Href: href}, 4 * time.Minute, true}, - {"no expiry, later", lfsAction{Href: href}, 5 * time.Minute, false}, - {"expires_in", lfsAction{Href: href, ExpiresIn: 600}, 9 * time.Minute, true}, - {"expires_in, within the margin", lfsAction{Href: href, ExpiresIn: 600}, 10*time.Minute - 10*time.Second, false}, - {"expires_in too short to use", lfsAction{Href: href, ExpiresIn: 10}, 0, false}, - {"expires_at", lfsAction{Href: href, ExpiresAt: now.Add(20 * time.Minute).Format(time.RFC3339)}, 19 * time.Minute, true}, - {"expires_at, past", lfsAction{Href: href, ExpiresAt: now.Add(20 * time.Minute).Format(time.RFC3339)}, 20 * time.Minute, false}, - {"long expiry capped", lfsAction{Href: href, ExpiresIn: 86400}, 2 * time.Hour, false}, - {"relative href", lfsAction{Href: "/a"}, 0, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var d lfsDownloads - d.put("repo", object, tt.action, now) - if _, kept := d.get("repo", object, now.Add(tt.at)); kept != tt.kept { - t.Errorf("kept = %v, want %v", kept, tt.kept) - } - }) - } - - var d lfsDownloads - d.put("repo", object, lfsAction{Href: href}, now) - if _, kept := d.get("repo", lfsObject{OID: object.OID, Size: 4}, now); kept { - t.Error("kept for another size") - } - if _, kept := d.get("other", object, now); kept { - t.Error("kept for another repository") - } - for i := range gitLFSKeptDownloads + 10 { - d.put("repo", lfsObject{OID: fmt.Sprintf("%064x", i), Size: 1}, lfsAction{Href: href}, now) - } - if len(d.entries) > gitLFSKeptDownloads { - t.Errorf("%d entries kept, want at most %d", len(d.entries), gitLFSKeptDownloads) - } -} - -func TestGitLFSObjectOfAnotherSizeIsRefused(t *testing.T) { - u := newLFSUpstream(t) - h := newLFSHandler(t, u) - path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)-1) - - if w := lfsRequest(h, http.MethodGet, path, ""); w.Code == http.StatusOK { - t.Fatalf("GET at the wrong size: status %d, body %q", w.Code, w.Body) - } - if u.downloads.Load() != 1 || u.batches.Load() != 1 { - t.Errorf("downloads %d, batches %d; want one of each, without a retry", u.downloads.Load(), u.batches.Load()) - } -} - -func TestExactSizeBody(t *testing.T) { - for _, tt := range []struct { - body string - size int64 - ok bool - }{ - {"abcdef", 6, true}, - {"abcdef", 5, false}, - {"abcdef", 7, false}, - {"", 0, true}, - {"a", 0, false}, - } { - got, err := io.ReadAll(&exactSizeBody{ReadCloser: io.NopCloser(strings.NewReader(tt.body)), left: tt.size}) - if (err == nil) != tt.ok || (err != nil && !errors.Is(err, errDownloadSize)) { - t.Errorf("%q at size %d: err %v, want ok %v", tt.body, tt.size, err, tt.ok) - } - if int64(len(got)) > tt.size+1 { - t.Errorf("%q at size %d: read %d bytes", tt.body, tt.size, len(got)) - } - } -} - -func TestGitLFSObjectCopiedFromAnotherRepository(t *testing.T) { - u := newLFSUpstream(t) - h := newLFSHandler(t, u) - object := func(repo string) string { - return fmt.Sprintf("/up/%s.git/info/lfs/objects/%s/%d", repo, u.oid, len(lfsTestContent)) - } - - if w := lfsRequest(h, http.MethodGet, object("acme/widget"), ""); w.Code != http.StatusOK { - t.Fatalf("GET: status %d, body %s", w.Code, w.Body) - } - w := lfsRequest(h, http.MethodGet, object("fork/widget"), "") - if w.Code != http.StatusOK || w.Body.String() != lfsTestContent { - t.Fatalf("GET from the fork: status %d, body %q", w.Code, w.Body) - } - if u.downloads.Load() != 1 || u.batches.Load() != 2 { - t.Errorf("downloads %d, batches %d; want the fork's object copied after its own batch", u.downloads.Load(), u.batches.Load()) - } - cached, err := h.proxy.GetCachedArtifact(t.Context(), gitEcosystem, "up/fork/widget", u.oid, u.oid) - if err != nil || cached == nil { - t.Fatalf("fork's object not cached: %v, %v", cached, err) - } - _ = cached.Reader.Close() - - // A repository the upstream says lacks the object doesn't get it, though - // a copy is stored. - if w := lfsRequest(h, http.MethodGet, object("stranger/widget"), ""); w.Code != http.StatusNotFound { - t.Errorf("GET from a repository without the object: status %d, body %s; want 404", w.Code, w.Body) - } -} diff --git a/internal/handler/git_mirror.go b/internal/handler/git_mirror.go deleted file mode 100644 index 38be233..0000000 --- a/internal/handler/git_mirror.go +++ /dev/null @@ -1,149 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "log/slog" - "net/url" - "path" - "strings" - - "github.com/git-pkgs/proxy/internal/gitcache" -) - -// maxSubmoduleDepth bounds how deeply Mirror follows submodules of -// submodules. -const maxSubmoduleDepth = 8 - -// Mirror caches the repository at location, a host and path such as -// github.com/acme/widgets as a pkg:git PURL names it, from the upstream.git -// host serving it. With version, a full commit ID or a tag, it caches that -// commit too, and the commits its submodules pin, from the upstream.git -// hosts serving them. A submodule no upstream.git host serves is skipped. -// Clones through /git are then served from those copies. -func (h *GitHandler) Mirror(ctx context.Context, location, version string) (gitcache.MirrorResult, error) { - return h.mirror(ctx, location, version, map[string]bool{}, 0) -} - -func (h *GitHandler) mirror(ctx context.Context, location, version string, seen map[string]bool, depth int) (gitcache.MirrorResult, error) { - repo, err := h.mirrorRepository(location) - if err != nil { - return gitcache.MirrorResult{}, err - } - result, err := h.cache.Mirror(ctx, repo.key, repo.url, version, repo.upstream.fetchWants) - if err != nil { - return result, err - } - for _, sub := range result.Submodules { - subLocation, err := gitSubmoduleLocation(repo.url, sub.URL) - if err == nil { - _, err = h.mirrorRepository(subLocation) - } - if err != nil { - h.logger().Warn("git mirror: skipping submodule", "repository", location, "submodule", sub.Path, "url", sub.URL, "error", err) - continue - } - if seen[subLocation+"@"+sub.Commit] { - continue - } - seen[subLocation+"@"+sub.Commit] = true - if depth >= maxSubmoduleDepth { - return result, fmt.Errorf("submodule %s: nested more than %d deep", sub.Path, maxSubmoduleDepth) - } - subResult, err := h.mirror(ctx, subLocation, sub.Commit, seen, depth+1) - if err != nil { - return result, fmt.Errorf("submodule %s: %w", sub.Path, err) - } - result.Cached = result.Cached && subResult.Cached - result.Bytes += subResult.Bytes - } - return result, nil -} - -func (h *GitHandler) logger() *slog.Logger { - if h.proxy != nil && h.proxy.Logger != nil { - return h.proxy.Logger - } - return slog.Default() -} - -// gitSubmoduleLocation turns the URL .gitmodules gives a submodule into a -// location as Mirror takes it. A relative URL is resolved against the -// superproject's repoURL, as git does; ssh, git and scp-like URLs name the -// host and path the repository has there. -func gitSubmoduleLocation(repoURL, submoduleURL string) (string, error) { - var host, repoPath string - switch { - case strings.HasPrefix(submoduleURL, "./"), strings.HasPrefix(submoduleURL, "../"): - u, err := url.Parse(repoURL) - if err != nil { - return "", err - } - host, repoPath = u.Host, path.Join(u.Path, submoduleURL) - case strings.Contains(submoduleURL, "://"): - u, err := url.Parse(submoduleURL) - if err != nil { - return "", err - } - switch u.Scheme { - case "http", "https": - host = u.Host - case "ssh", "git", "git+ssh", "ssh+git": - host = u.Hostname() - default: - return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) - } - repoPath = u.Path - default: - // scp-like: [user@]host:path - hostPart, p, ok := strings.Cut(submoduleURL, ":") - if !ok || strings.Contains(hostPart, "/") { - return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) - } - _, host, _ = strings.Cut(hostPart, "@") - if host == "" { - host = hostPart - } - repoPath = p - } - repoPath = strings.Trim(path.Clean("/"+repoPath), "/") - if host == "" || repoPath == "" { - return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) - } - return strings.ToLower(host) + "/" + repoPath, nil -} - -// mirrorRepository resolves location to a repository of the upstream.git -// host whose URL it starts with, the longest such URL if several do. -func (h *GitHandler) mirrorRepository(location string) (gitRepository, error) { - var best, bestPrefix, rest string - for name, upstream := range h.upstreams { - prefix := gitLocationPrefix(upstream.URL) - path, ok := strings.CutPrefix(location, prefix) - if prefix == "" || !ok || path == "" { - continue - } - if len(prefix) > len(bestPrefix) || (len(prefix) == len(bestPrefix) && name < best) { - best, bestPrefix, rest = name, prefix, path - } - } - if best == "" { - return gitRepository{}, fmt.Errorf("no upstream.git host serves %s", location) - } - repo, status, msg := h.repository(best + "/" + rest) - if status != 0 { - return gitRepository{}, fmt.Errorf("%s: %s", location, msg) - } - return repo, nil -} - -// gitLocationPrefix is how a pkg:git PURL names the repositories under an -// upstream URL: its host and path, ending in a slash. Empty for a URL -// without a host. -func gitLocationPrefix(upstreamURL string) string { - u, err := url.Parse(upstreamURL) - if err != nil || u.Host == "" { - return "" - } - return strings.ToLower(u.Host) + strings.TrimSuffix(u.Path, "/") + "/" -} diff --git a/internal/handler/git_mirror_test.go b/internal/handler/git_mirror_test.go deleted file mode 100644 index 227bd71..0000000 --- a/internal/handler/git_mirror_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package handler - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/git-pkgs/proxy/internal/config" -) - -func TestGitMirrorLocation(t *testing.T) { - h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ - "github": {URL: "https://github.com"}, - "wikimedia": {URL: "https://gitlab.wikimedia.org", Repos: []string{"mediawiki/**"}}, - "sub": {URL: "https://git.example/r/"}, - "nested": {URL: "https://git.example/r/deep"}, - }}) - tests := []struct { - location string - ok bool - key, repoURL string - }{ - {"github.com/acme/widgets", true, h.upstreams["github"].dir + "/acme/widgets.git", "https://github.com/acme/widgets"}, - {"github.com/acme/widgets.git", true, h.upstreams["github"].dir + "/acme/widgets.git", "https://github.com/acme/widgets.git"}, - {"gitlab.wikimedia.org/mediawiki/skins/Vector", true, h.upstreams["wikimedia"].dir + "/mediawiki/skins/Vector.git", "https://gitlab.wikimedia.org/mediawiki/skins/Vector"}, - {"gitlab.wikimedia.org/other/repo", false, "", ""}, - {"git.example/r/a", true, h.upstreams["sub"].dir + "/a.git", "https://git.example/r/a"}, - // The longest matching upstream URL wins. - {"git.example/r/deep/a", true, h.upstreams["nested"].dir + "/a.git", "https://git.example/r/deep/a"}, - {"git.example/other", false, "", ""}, - {"git.example/r", false, "", ""}, - {"github.com.evil/acme/widgets", false, "", ""}, - {"github.com/acme/../widgets", false, "", ""}, - {"codeberg.org/forgejo/forgejo", false, "", ""}, - } - for _, tt := range tests { - t.Run(tt.location, func(t *testing.T) { - repo, err := h.mirrorRepository(tt.location) - if (err == nil) != tt.ok { - t.Fatalf("error %v, want ok %v", err, tt.ok) - } - if tt.ok && (repo.key != tt.key || repo.url != tt.repoURL) { - t.Errorf("key %q, URL %q; want %q, %q", repo.key, repo.url, tt.key, tt.repoURL) - } - }) - } -} - -func TestGitMirrorCachesCommitForClones(t *testing.T) { - g := newGitTestSetup(t, 0, 0) - sha := runGitCommand(t, g.work, "rev-parse", "HEAD") - location := strings.TrimPrefix(g.upstreamURL, "http://") + "/repo" - - got, err := g.handler.Mirror(context.Background(), location, sha) - if err != nil { - t.Fatalf("Mirror: %v", err) - } - if got.Cached { - t.Errorf("first mirror = %+v, want fetched", got) - } - got, err = g.handler.Mirror(context.Background(), location, sha) - if err != nil || !got.Cached { - t.Errorf("second mirror = %+v, %v, want cached", got, err) - } - - // With the upstream gone, a clone through the proxy gets the mirrored copy. - g.upstream.Close() - clone := t.TempDir() - runGitCommand(t, "", "clone", "--quiet", g.url(), clone) - if head := runGitCommand(t, clone, "rev-parse", "HEAD"); head != sha { - t.Errorf("cloned HEAD %s, want %s", head, sha) - } -} - -func TestGitSubmoduleLocation(t *testing.T) { - tests := []struct { - repoURL, submoduleURL, want string - }{ - {"https://github.com/acme/widgets", "../lib.git", "github.com/acme/lib.git"}, - {"https://github.com/acme/widgets.git", "../../other/lib", "github.com/other/lib"}, - {"https://github.com/acme/widgets", "./vendored", "github.com/acme/widgets/vendored"}, - {"http://127.0.0.1:8080/team/repo", "../lib.git", "127.0.0.1:8080/team/lib.git"}, - {"https://github.com/acme/widgets", "https://GitLab.com/group/lib.git/", "gitlab.com/group/lib.git"}, - {"https://github.com/acme/widgets", "ssh://git@github.com:22/acme/lib.git", "github.com/acme/lib.git"}, - {"https://github.com/acme/widgets", "git://git.example/lib", "git.example/lib"}, - {"https://github.com/acme/widgets", "git@github.com:acme/lib.git", "github.com/acme/lib.git"}, - {"https://github.com/acme/widgets", "/srv/git/lib.git", ""}, - {"https://github.com/acme/widgets", "file:///srv/git/lib.git", ""}, - {"https://github.com/acme/widgets", "", ""}, - } - for _, tt := range tests { - got, err := gitSubmoduleLocation(tt.repoURL, tt.submoduleURL) - if tt.want == "" { - if err == nil { - t.Errorf("location of %q = %q, want an error", tt.submoduleURL, got) - } - continue - } - if err != nil || got != tt.want { - t.Errorf("location of %q from %s = %q, %v; want %q", tt.submoduleURL, tt.repoURL, got, err, tt.want) - } - } -} - -// TestGitMirrorCachesSubmodules mirrors a commit with a submodule served by -// the same upstream and one on a host no upstream.git serves, which is -// skipped. -func TestGitMirrorCachesSubmodules(t *testing.T) { - g := newGitTestSetup(t, 0, 0) - libBare := filepath.Join(g.root, "lib.git") - runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", libBare) - libWork := t.TempDir() - runGitCommand(t, "", "clone", "--quiet", libBare, libWork) - runGitCommand(t, libWork, "commit", "--quiet", "--allow-empty", "-m", "lib") - runGitCommand(t, libWork, "push", "--quiet", "origin", "HEAD:main") - libCommit := runGitCommand(t, libWork, "rev-parse", "HEAD") - - gitmodules := "[submodule \"lib\"]\n\tpath = lib\n\turl = ../lib.git\n" + - "[submodule \"elsewhere\"]\n\tpath = elsewhere\n\turl = https://elsewhere.example/x.git\n" - if err := os.WriteFile(filepath.Join(g.work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { - t.Fatal(err) - } - runGitCommand(t, g.work, "add", ".gitmodules") - runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",lib") - runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+strings.Repeat("ab", 20)+",elsewhere") - runGitCommand(t, g.work, "commit", "--quiet", "-m", "submodules") - runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") - sha := runGitCommand(t, g.work, "rev-parse", "HEAD") - location := strings.TrimPrefix(g.upstreamURL, "http://") + "/repo" - - got, err := g.handler.Mirror(context.Background(), location, sha) - if err != nil { - t.Fatalf("Mirror: %v", err) - } - if got.Cached { - t.Errorf("first mirror = %+v, want fetched", got) - } - if got, err = g.handler.Mirror(context.Background(), location, sha); err != nil || !got.Cached { - t.Errorf("second mirror = %+v, %v, want cached", got, err) - } - if !strings.Contains(g.logs.String(), "elsewhere.example") { - t.Errorf("log doesn't name the skipped submodule:\n%s", g.logs.String()) - } - - // With the upstream gone, the submodule's commit is served from the cache. - g.upstream.Close() - clone := t.TempDir() - runGitCommand(t, "", "clone", "--quiet", g.server.URL+"/git/up/lib", clone) - if head := runGitCommand(t, clone, "rev-parse", "HEAD"); head != libCommit { - t.Errorf("cloned lib HEAD %s, want %s", head, libCommit) - } -} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go deleted file mode 100644 index aeec2bc..0000000 --- a/internal/handler/git_test.go +++ /dev/null @@ -1,341 +0,0 @@ -package handler - -import ( - "bufio" - "fmt" - "io" - "log/slog" - "math/rand/v2" - "net" - "net/http" - "net/http/cgi" - "net/http/httptest" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/gitcache" -) - -func TestParseGitRequest(t *testing.T) { - h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ - "up": {URL: "https://git.example/r/"}, - "limited": {URL: "https://git.example", Repos: []string{"allowed/*"}}, - }}) - up, limited := h.upstreams["up"].dir, h.upstreams["limited"].dir - tests := []struct { - method, target string - status int - key, repoURL string - }{ - {"GET", "/up/repo.git/info/refs?service=git-upload-pack", 0, up + "/repo.git", "https://git.example/r/repo.git"}, - {"GET", "/up/a/b/info/refs?service=git-upload-pack", 0, up + "/a/b.git", "https://git.example/r/a/b"}, - {"POST", "/up/a/b%20c/git-upload-pack", 0, up + "/a/b c.git", "https://git.example/r/a/b%20c"}, - {"GET", "/up/a.git/b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, - {"GET", "/up/a/../b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, - {"GET", "/up/a//b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, - {"GET", "/up/.git/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, - {"GET", "/other/repo/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, - {"GET", "/up/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, - {"GET", "/up/repo/info/refs?service=git-receive-pack", http.StatusForbidden, "", ""}, - {"POST", "/up/repo/git-receive-pack", http.StatusForbidden, "", ""}, - {"GET", "/up/repo/info/refs", http.StatusForbidden, "", ""}, - {"GET", "/up/repo/objects/info/packs", http.StatusNotFound, "", ""}, - {"POST", "/up/repo/info/refs?service=git-upload-pack", http.StatusMethodNotAllowed, "", ""}, - {"GET", "/up/repo/git-upload-pack", http.StatusMethodNotAllowed, "", ""}, - {"GET", "/limited/allowed/repo/info/refs?service=git-upload-pack", 0, limited + "/allowed/repo.git", "https://git.example/allowed/repo"}, - {"GET", "/limited/other/repo/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, - } - for _, tt := range tests { - t.Run(tt.method+" "+tt.target, func(t *testing.T) { - r := httptest.NewRequest(tt.method, tt.target, nil) - r.Header.Set("Content-Type", gitRequestType) - req, status, _ := h.parse(r) - if status != tt.status { - t.Fatalf("status %d, want %d", status, tt.status) - } - if status == 0 && (req.key != tt.key || req.repoURL != tt.repoURL) { - t.Errorf("key %q, URL %q; want %q, %q", req.key, req.repoURL, tt.key, tt.repoURL) - } - }) - } - - r := httptest.NewRequest(http.MethodPost, "/up/repo/git-upload-pack", nil) - if _, status, _ := h.parse(r); status != http.StatusUnsupportedMediaType { - t.Errorf("POST without the request content type: status %d, want 415", status) - } -} - -// TestGitUpstreamCacheDirectory checks the cache directory follows the -// upstream's URL, so repointing an upstream does not serve the old host's -// repositories. -func TestGitUpstreamCacheDirectory(t *testing.T) { - dir := func(url string) string { - h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{"up": {URL: url}}}) - return h.upstreams["up"].dir - } - if a, b := dir("https://git.example"), dir("https://other.example"); a == b || !strings.HasPrefix(a, "up-") { - t.Errorf("directories %q and %q, want distinct ones named after the upstream", a, b) - } - if a, b := dir("https://git.example"), dir("https://git.example/"); a != b { - t.Errorf("a trailing slash changed the directory: %q, %q", a, b) - } -} - -// TestGitFetchByIDOffWithCredentials checks commits are not fetched by ID -// for an upstream that upstream.auth has credentials for: there a fetch by -// ID could reach repositories outside its repos allowlist. -func TestGitFetchByIDOffWithCredentials(t *testing.T) { - h := NewGitHandler(nil, nil, config.UpstreamConfig{ - Git: map[string]config.GitUpstream{ - "public": {URL: "https://github.com"}, - "private": {URL: "https://git.example", Repos: []string{"team/*"}}, - "scoped": {URL: "https://other.example", Repos: []string{"team/*"}}, - }, - Auth: map[string]config.AuthConfig{ - "https://git.example": {Type: "bearer", Token: "t"}, - "https://other.example/team": {Type: "bearer", Token: "t"}, - }, - }) - for name, want := range map[string]bool{"public": true, "private": false, "scoped": false} { - if got := h.upstreams[name].fetchWants; got != want { - t.Errorf("%s: fetchWants = %v, want %v", name, got, want) - } - } -} - -func runGitCommand(t *testing.T, dir string, args ...string) string { - t.Helper() - cmd := exec.Command("git", args...) - cmd.Dir = dir - cmd.Env = []string{ - "PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", - "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", - "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", - } - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) - } - return strings.TrimSpace(string(out)) -} - -// gitTestSetup is a GitHandler in front of a cache of one upstream -// repository, "repo", served by git http-backend. -type gitTestSetup struct { - work string - // root holds the upstream's bare repositories. - root string - server *httptest.Server - // upstream serves the repository, at upstreamURL/repo, and handler is - // the GitHandler under test. - upstream *httptest.Server - upstreamURL string - handler *GitHandler - logs *syncLog - // slow delays every upstream request by half a second. - slow atomic.Bool - // active counts the requests the handler is serving. - active atomic.Int32 -} - -// newGitTestSetup serves the cache with the given server read timeout and, -// unless zero, GitHandler.writeStall. -func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTestSetup { - t.Helper() - execPath := runGitCommand(t, "", "--exec-path") - backendPath := "" - for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { - if path := filepath.Join(execPath, name); fileExists(path) { - backendPath = path - } - } - if backendPath == "" { - t.Skip("git http-backend is not installed") - } - root := t.TempDir() - bare := filepath.Join(root, "repo.git") - runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) - runGitCommand(t, bare, "config", "uploadpack.allowAnySHA1InWant", "true") - g := &gitTestSetup{work: t.TempDir(), root: root, logs: &syncLog{}} - runGitCommand(t, "", "clone", "--quiet", bare, g.work) - runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "first") - runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") - - backend := &cgi.Handler{ - Path: backendPath, - Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}, - } - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if g.slow.Load() { - time.Sleep(500 * time.Millisecond) - } - backend.ServeHTTP(w, r) - })) - t.Cleanup(upstream.Close) - g.upstream, g.upstreamURL = upstream, upstream.URL - - logger := slog.New(slog.NewTextHandler(g.logs, nil)) - cache, err := gitcache.New(gitcache.Options{ - CacheDir: t.TempDir(), - RefreshWait: 5 * time.Second, - StaleMaxAge: time.Hour, - Client: &http.Client{}, - Logger: logger, - Upstreams: map[string]string{"up": gitcache.UpstreamDir("up", upstream.URL)}, - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = cache.Close() }) - h := NewGitHandler(&Proxy{Logger: logger}, cache, config.UpstreamConfig{Git: map[string]config.GitUpstream{"up": {URL: upstream.URL}}}) - if writeStall > 0 { - h.writeStall = writeStall - } - g.handler = h - routes := http.StripPrefix("/git", h.Routes()) - g.server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - g.active.Add(1) - defer g.active.Add(-1) - routes.ServeHTTP(w, r) - })) - g.server.Config.ReadTimeout = readTimeout - g.server.Start() - t.Cleanup(g.server.Close) - return g -} - -func (g *gitTestSetup) url() string { - return g.server.URL + "/git/up/repo" -} - -type syncLog struct { - mu sync.Mutex - buf strings.Builder -} - -func (l *syncLog) Write(p []byte) (int, error) { - l.mu.Lock() - defer l.mu.Unlock() - return l.buf.Write(p) -} - -func (l *syncLog) String() string { - l.mu.Lock() - defer l.mu.Unlock() - return l.buf.String() -} - -// TestGitRequestBodyOutlastsServerReadTimeout sends a protocol v0 request -// whose haves come after a want the cache must fetch first, so upload-pack -// reads them after the server's read timeout has passed. -func TestGitRequestBodyOutlastsServerReadTimeout(t *testing.T) { - g := newGitTestSetup(t, 200*time.Millisecond, 0) - runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "pull request") - runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:refs/pull/1/head") - sha := runGitCommand(t, g.work, "rev-parse", "HEAD") - runGitCommand(t, "", "clone", "--quiet", g.url(), filepath.Join(t.TempDir(), "clone")) - - var body strings.Builder - body.WriteString(pktLine("want "+sha+" side-band-64k\n") + "0000") - for i := range 2000 { - body.WriteString(pktLine(fmt.Sprintf("have %040x\n", i+1))) - } - body.WriteString(pktLine("done\n")) - - g.slow.Store(true) - resp, err := http.Post(g.url()+"/git-upload-pack", gitRequestType, strings.NewReader(body.String())) - if err != nil { - t.Fatal(err) - } - defer func() { _ = resp.Body.Close() }() - out, _ := io.ReadAll(resp.Body) - if resp.StatusCode != http.StatusOK || !strings.Contains(string(out), "PACK") { - t.Errorf("status %d, no pack in %d bytes: %.200q", resp.StatusCode, len(out), out) - } -} - -// TestGitSlowRequestBodyStartsNothing sends part of a request body and then -// nothing. The request ends at the server's read timeout, without an -// upload-pack waiting for the rest. -func TestGitSlowRequestBodyStartsNothing(t *testing.T) { - g := newGitTestSetup(t, 200*time.Millisecond, 0) - runGitCommand(t, "", "clone", "--quiet", g.url(), filepath.Join(t.TempDir(), "clone")) - - conn, err := net.Dial("tcp", g.server.Listener.Addr().String()) - if err != nil { - t.Fatal(err) - } - defer func() { _ = conn.Close() }() - part := pktLine("command=ls-refs\n") - _, err = fmt.Fprintf(conn, "POST /git/up/repo/git-upload-pack HTTP/1.1\r\nHost: proxy\r\n"+ - "Content-Type: %s\r\nGit-Protocol: version=2\r\nContent-Length: 100\r\n\r\n%s", gitRequestType, part) - if err != nil { - t.Fatal(err) - } - _ = conn.SetReadDeadline(time.Now().Add(10 * time.Second)) - resp, err := http.ReadResponse(bufio.NewReader(conn), nil) - if err != nil { - t.Fatalf("no answer to a request whose body stopped: %v", err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusBadRequest { - t.Errorf("status %d, want %d", resp.StatusCode, http.StatusBadRequest) - } -} - -// TestGitStalledClientIsDropped fetches a pack larger than the connection -// buffers and never reads it. The request ends once a write has waited -// writeStall, rather than holding upload-pack. -func TestGitStalledClientIsDropped(t *testing.T) { - g := newGitTestSetup(t, 0, 200*time.Millisecond) - data := make([]byte, 16<<20) - _, _ = rand.NewChaCha8([32]byte{}).Read(data) - if err := os.WriteFile(filepath.Join(g.work, "large"), data, 0o600); err != nil { - t.Fatal(err) - } - runGitCommand(t, g.work, "add", "large") - runGitCommand(t, g.work, "commit", "--quiet", "-m", "large") - runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") - sha := runGitCommand(t, g.work, "rev-parse", "HEAD") - runGitCommand(t, "", "clone", "--quiet", g.url(), filepath.Join(t.TempDir(), "clone")) - - conn, err := net.Dial("tcp", g.server.Listener.Addr().String()) - if err != nil { - t.Fatal(err) - } - defer func() { _ = conn.Close() }() - body := pktLine("want "+sha+" side-band-64k\n") + "0000" + pktLine("done\n") - _, err = fmt.Fprintf(conn, "POST /git/up/repo/git-upload-pack HTTP/1.1\r\nHost: proxy\r\n"+ - "Content-Type: %s\r\nContent-Length: %d\r\n\r\n%s", gitRequestType, len(body), body) - if err != nil { - t.Fatal(err) - } - for deadline := time.Now().Add(10 * time.Second); g.active.Load() == 0; { - if time.Now().After(deadline) { - t.Fatal("request never reached the handler") - } - time.Sleep(10 * time.Millisecond) - } - for deadline := time.Now().Add(10 * time.Second); g.active.Load() != 0; { - if time.Now().After(deadline) { - t.Fatal("the handler still serves a client that stopped reading") - } - time.Sleep(10 * time.Millisecond) - } -} - -func pktLine(payload string) string { - return fmt.Sprintf("%04x%s", len(payload)+4, payload) -} - -func fileExists(path string) bool { - _, err := os.Stat(path) - return err == nil -} diff --git a/internal/handler/go.go b/internal/handler/go.go index 08dae57..955a89c 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -1,20 +1,9 @@ package handler import ( - "crypto/sha256" - "encoding/hex" - "errors" "fmt" "net/http" - "net/url" "strings" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries/fetch" - gomodule "golang.org/x/mod/module" ) const ( @@ -22,24 +11,11 @@ const ( asciiCaseOffset = 32 // difference between lowercase and uppercase ASCII letters ) -func init() { - retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: golangRetentionKey}) -} - -func golangRetentionKey(p *purl.PURL) (string, bool) { - key, ok := retention.DefaultCanonical("golang")(p) - return packageurl.PolicyPackageKey(key), ok -} - // GoHandler handles Go module proxy protocol requests. type GoHandler struct { - proxy *Proxy - upstreamURL string - proxyURL string - sumDBs map[string]string - sumDBOffline bool - sumDBProbes goSumDBProbes - vulnDBURL string + proxy *Proxy + upstreamURL string + proxyURL string } // NewGoHandler creates a new Go module proxy handler. @@ -48,25 +24,14 @@ func NewGoHandler(proxy *Proxy, proxyURL string) *GoHandler { proxy: proxy, upstreamURL: goUpstream, proxyURL: strings.TrimSuffix(proxyURL, "/"), - sumDBs: map[string]string{defaultGoSumDBName: defaultGoSumDBURL}, - vulnDBURL: defaultGoVulnDBURL, } } -// NewGoHandlerWithUpstream creates a Go module handler with a custom upstream. -func NewGoHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *GoHandler { - h := NewGoHandler(proxy, proxyURL) - h.upstreamURL = configuredUpstreamURL(upstreamURL, goUpstream) - return h -} - // Routes returns the HTTP handler for Go proxy requests. func (h *GoHandler) Routes() http.Handler { // Go module paths can contain slashes, so just use the handler directly return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // govulncheck sends HEAD for index/modules.json.gz to check that a - // -db other than vuln.go.dev is a vulnerability database. - if r.Method != http.MethodGet && (r.Method != http.MethodHead || !strings.HasPrefix(r.URL.Path, "/vulndb/")) { + if r.Method != http.MethodGet { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } @@ -78,17 +43,9 @@ func (h *GoHandler) Routes() http.Handler { func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { path := strings.TrimPrefix(r.URL.Path, "/") - // Checksum database requests go to the database, not the module upstream - if rest, ok := strings.CutPrefix(path, "sumdb/"); ok { - h.handleSumDB(w, r, rest) - return - } - - // So do vulnerability database requests, which govulncheck sends when - // -db points at /go/vulndb. A module path's first element holds a - // dot, so neither prefix can be a module. - if rest, ok := strings.CutPrefix(path, "vulndb/"); ok { - h.handleVulnDB(w, r, rest) + // Sumdb requests - proxy through + if strings.HasPrefix(path, "sumdb/") { + h.proxyUpstream(w, r) return } @@ -98,33 +55,18 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { rest := path[idx+4:] // after "/@v/" decodedMod := decodeGoModule(module) - cooldown := h.cooldownEnabled() switch { - case rest == "list" && cooldown: - h.serveCooldownList(w, r, module, decodedMod) - case rest == "list": // GET /{module}/@v/list - list versions h.proxyCached(w, r, decodedMod+"/@v/list") - case strings.HasSuffix(rest, ".info") && cooldown: - h.serveCooldownInfo(w, r, module, decodedMod, rest) - case strings.HasSuffix(rest, ".info"): // GET /{module}/@v/{version}.info - version metadata - h.proxyVersionFile(w, r, decodedMod+"/@v/"+rest, strings.TrimSuffix(rest, ".info")) + h.proxyCached(w, r, decodedMod+"/@v/"+rest) case strings.HasSuffix(rest, ".mod"): // GET /{module}/@v/{version}.mod - go.mod file - version, err := gomodule.UnescapeVersion(strings.TrimSuffix(rest, ".mod")) - if err != nil { - http.NotFound(w, r) - return - } - if h.checkCooldown(w, r, module, decodedMod, version) { - return - } - h.proxyVersionFile(w, r, decodedMod+"/@v/"+rest, strings.TrimSuffix(rest, ".mod")) + h.proxyCached(w, r, decodedMod+"/@v/"+rest) case strings.HasSuffix(rest, ".zip"): // GET /{module}/@v/{version}.zip - source archive (cache this) @@ -140,10 +82,6 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { // Check for @latest if strings.HasSuffix(path, "/@latest") { module := strings.TrimSuffix(path, "/@latest") - if h.cooldownEnabled() { - h.serveCooldownLatest(w, r, module, decodeGoModule(module)) - return - } h.proxyCached(w, r, decodeGoModule(module)+"/@latest") return } @@ -155,128 +93,24 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, module, version string) { // Decode module path (! followed by lowercase = uppercase) decodedModule := decodeGoModule(module) - filename := goZipFilename(decodedModule, version) + filename := fmt.Sprintf("%s@%s.zip", lastComponent(decodedModule), version) - h.proxy.Logger.Debug("go module download request", + h.proxy.Logger.Info("go module download request", "module", decodedModule, "version", version) - // The URL escapes uppercase letters in the version (v1.0.0-!r!c1 for - // v1.0.0-RC1), and the zip is cached under that form, so the denylist - // is checked against the version itself. - // A version that doesn't unescape can't be checked, so it isn't fetched. - unescaped, err := gomodule.UnescapeVersion(version) + result, err := h.proxy.GetOrFetchArtifact(r.Context(), "golang", decodedModule, version, filename) if err != nil { - http.NotFound(w, r) - return - } - if h.proxy.versionDenied("golang", decodedModule, unescaped) { - http.Error(w, ErrVersionDenied.Error()+": "+canonicalVersionPURL("golang", decodedModule, unescaped), http.StatusForbidden) - return - } - if h.checkCooldown(w, r, module, decodedModule, unescaped) { - return - } - - downloadURL := h.upstreamURL + r.URL.Path - h.discardOtherModulesZip(decodedModule, module, version, filename) - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), "golang", decodedModule, version, filename, downloadURL, - ) - if err != nil { - if errors.Is(err, fetch.ErrNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if errors.Is(err, ErrArtifactBlocked) || errors.Is(err, ErrVersionDenied) { - http.Error(w, err.Error(), http.StatusForbidden) - return - } - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("failed to get artifact", "error", err) http.Error(w, "failed to fetch module", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) + ServeArtifact(w, result) } -// Keep the module spelling hash used by caches with lowercased PURL keys. -func goZipFilename(module, version string) string { - base := lastComponent(module) + "@" + version - if module == strings.ToLower(module) { - return base + ".zip" - } - digest := sha256.Sum256([]byte(module)) - return base + "~" + hex.EncodeToString(digest[:goModuleHashBytes]) + ".zip" -} - -// goModuleHashBytes is how much of a module path's hash a cached zip's -// filename carries: enough to tell apart paths that differ only in case. -const goModuleHashBytes = 8 - -// An old cache key may combine case-distinct modules. The upstream path -// identifies which module owns an individual archive. -func (p *Proxy) rekeyLegacyGoArtifact(pkgPURL, versionPURL, filename string) (bool, error) { - if !strings.HasPrefix(versionPURL, "pkg:golang/") { - return false, nil - } - parsed, err := purl.Parse(versionPURL) - if err != nil || parsed.FullName() == strings.ToLower(parsed.FullName()) { - return false, nil - } - name := parsed.FullName() - _, legacyVersion, err := packagePURLStrings("golang", strings.ToLower(name), parsed.Version) - if err != nil { - return false, err - } - record, err := p.DB.GetArtifact(legacyVersion, filename) - if err != nil { - return false, fmt.Errorf("reading legacy Go artifact: %w", err) - } - if record == nil || !record.StoragePath.Valid || record.Source != database.SourceUpstream { - return false, nil - } - escaped, err := gomodule.EscapePath(name) - if err != nil { - return false, nil - } - upstream, err := url.Parse(record.UpstreamURL) - if err != nil || !strings.HasSuffix(upstream.Path, "/"+escaped+"/@v/"+parsed.Version+".zip") { - return false, nil - } - return p.DB.RekeyCachedArtifact(record, &database.Package{PURL: pkgPURL, Ecosystem: "golang", Name: name}, - &database.Version{PURL: versionPURL, PackagePURL: pkgPURL}) -} - -// discardOtherModulesZip drops a cached zip that another module stored -// under filename. Before module paths kept their case, a module with -// uppercase letters was cached under the plain filename of its lowercase -// twin, and whichever came first was served for both. The upstream URL a -// zip was fetched from names its module in escaped form, so a record whose -// URL names a different module is discarded and fetched again. Only the -// module path is compared, so pointing upstream.go elsewhere keeps the cache. -func (h *GoHandler) discardOtherModulesZip(module, escapedModule, version, filename string) { - _, versionPURL, err := packagePURLStrings("golang", module, version) - if err != nil { - return - } - record, err := h.proxy.DB.GetArtifact(versionPURL, filename) - if err != nil || record == nil || !record.StoragePath.Valid || record.Source == database.SourceLocal { - return - } - // Only a URL that names another Go module proves the zip is someone - // else's; a record of unknown origin is kept. - if !strings.Contains(record.UpstreamURL, "/@v/") || - strings.HasSuffix(record.UpstreamURL, "/"+escapedModule+"/@v/"+version+".zip") { - return - } - h.proxy.Logger.Warn("cached zip belongs to a module that differs only in case, discarding", - "module", module, "version", version, "cached_from", record.UpstreamURL) - if err := h.proxy.DB.DiscardArtifact(versionPURL, filename, record.StoragePath.String); err != nil { - h.proxy.Logger.Warn("failed to discard cached zip", "module", module, "version", version, "error", err) - } +// proxyUpstream forwards a request to proxy.golang.org without caching. +func (h *GoHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { + h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, nil) } // proxyCached forwards a request with metadata caching. @@ -284,29 +118,6 @@ func (h *GoHandler) proxyCached(w http.ResponseWriter, r *http.Request, cacheKey h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "golang", cacheKey, "*/*") } -// proxyVersionFile serves a version's .info or .mod. For a canonical version -// it never changes, so it is served from the cache without revalidation. -func (h *GoHandler) proxyVersionFile(w http.ResponseWriter, r *http.Request, cacheKey, escapedVersion string) { - if goVersionImmutable(escapedVersion) { - h.proxy.proxyImmutable(w, r, h.upstreamURL+r.URL.Path, "golang", cacheKey, "*/*") - return - } - h.proxyCached(w, r, cacheKey) -} - -// goVersionImmutable reports whether escapedVersion, as it appears in a -// proxy URL, names a canonical version, whose .info and .mod the module -// proxy protocol and the checksum database fix for good. This includes -// pseudo-versions, which name a commit by its hash, and +incompatible -// versions. Anything else, such as a branch name or v1.2 for v1.2.0, is a -// query that can resolve to another version later and stays on the TTL. -// The .zip of a canonical version is cached for good already, and a .mod -// that changed upstream would no longer match it. -func goVersionImmutable(escapedVersion string) bool { - version, err := gomodule.UnescapeVersion(escapedVersion) - return err == nil && version != "" && gomodule.CanonicalVersion(version) == version -} - // decodeGoModule decodes an encoded module path. // In the encoding, uppercase letters are represented as "!" followed by lowercase. func decodeGoModule(encoded string) string { diff --git a/internal/handler/go_case_test.go b/internal/handler/go_case_test.go deleted file mode 100644 index 91bac44..0000000 --- a/internal/handler/go_case_test.go +++ /dev/null @@ -1,204 +0,0 @@ -package handler - -import ( - "context" - "io" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/registries/fetch" -) - -const ( - upperLogrusPath = "/github.com/!sirupsen/logrus/@v/v1.0.6.zip" - lowerLogrusPath = "/github.com/sirupsen/logrus/@v/v1.0.6.zip" -) - -func TestGoZipFilenameKeepsCase(t *testing.T) { - lower := goZipFilename("github.com/sirupsen/logrus", "v1.0.6") - if lower != "logrus@v1.0.6.zip" { - t.Errorf("lowercase module filename = %q, want the plain name", lower) - } - upper := goZipFilename("github.com/Sirupsen/logrus", "v1.0.6") - if upper == lower || !strings.HasPrefix(upper, "logrus@v1.0.6~") || !strings.HasSuffix(upper, ".zip") { - t.Errorf("uppercase module filename = %q, want logrus@v1.0.6~{hash}.zip", upper) - } - if other := goZipFilename("github.com/SIRUPSEN/logrus", "v1.0.6"); other == upper { - t.Errorf("differently cased modules share filename %q", other) - } -} - -// goCaseUpstream serves different bytes for each module path and counts -// requests per path. -func goCaseUpstream(t *testing.T) (*httptest.Server, map[string]*atomic.Int32) { - t.Helper() - counts := map[string]*atomic.Int32{upperLogrusPath: {}, lowerLogrusPath: {}} - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - count, ok := counts[r.URL.Path] - if !ok { - http.NotFound(w, r) - return - } - count.Add(1) - _, _ = io.WriteString(w, "zip of "+r.URL.Path) - })) - t.Cleanup(srv.Close) - return srv, counts -} - -func newGoCaseHandler(t *testing.T, upstream *httptest.Server) (http.Handler, *Proxy) { - t.Helper() - proxy, _, store, _ := setupTestProxy(t) - store.seekable = true - proxy.HTTPClient = upstream.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - return NewGoHandlerWithUpstream(proxy, "http://localhost", upstream.URL).Routes(), proxy -} - -func getGo(h http.Handler, path string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - return w -} - -func TestGoModulesDifferingInCaseAreCachedApart(t *testing.T) { - upstream, counts := goCaseUpstream(t) - h, _ := newGoCaseHandler(t, upstream) - - for range 2 { - for _, path := range []string{upperLogrusPath, lowerLogrusPath} { - if w := getGo(h, path); w.Code != http.StatusOK || w.Body.String() != "zip of "+path { - t.Fatalf("GET %s: status = %d, body = %q", path, w.Code, w.Body) - } - } - } - for path, count := range counts { - if n := count.Load(); n != 1 { - t.Errorf("%s fetched %d times, want once and then cached", path, n) - } - } -} - -// Before module paths kept their case, the uppercase module was cached under -// the lowercase module's plain filename. The lowercase module must not be -// served those bytes. -func TestGoZipCachedForAnotherModuleIsRefetched(t *testing.T) { - upstream, counts := goCaseUpstream(t) - h, proxy := newGoCaseHandler(t, upstream) - seedGoZip(t, proxy, "https://proxy.golang.org"+upperLogrusPath, "zip of "+upperLogrusPath) - - if w := getGo(h, lowerLogrusPath); w.Code != http.StatusOK || w.Body.String() != "zip of "+lowerLogrusPath { - t.Fatalf("status = %d, body = %q, want the lowercase module's zip", w.Code, w.Body) - } - if n := counts[lowerLogrusPath].Load(); n != 1 { - t.Errorf("lowercase module fetched %d times, want 1", n) - } -} - -// A zip cached from another upstream host for the same module stays valid: -// only the module path decides whether a cached zip belongs to the request. -func TestGoZipFromAnotherUpstreamHostIsKept(t *testing.T) { - upstream, counts := goCaseUpstream(t) - h, proxy := newGoCaseHandler(t, upstream) - seedGoZip(t, proxy, "https://old-proxy.example"+lowerLogrusPath, "zip of "+lowerLogrusPath) - - if w := getGo(h, lowerLogrusPath); w.Code != http.StatusOK || w.Body.String() != "zip of "+lowerLogrusPath { - t.Fatalf("status = %d, body = %q", w.Code, w.Body) - } - if n := counts[lowerLogrusPath].Load(); n != 0 { - t.Errorf("zip fetched %d times, want it served from the cache", n) - } -} - -func TestGoLegacyCacheKeepsCaseDistinctModules(t *testing.T) { - upstream, counts := goCaseUpstream(t) - h, proxy := newGoCaseHandler(t, upstream) - seedGoZip(t, proxy, upstream.URL+lowerLogrusPath, "zip of "+lowerLogrusPath) - old := seedLegacyUpperGoZip(t, proxy, "https://old-proxy.example"+upperLogrusPath, "zip of "+upperLogrusPath) - upstream.Close() - - for range 2 { - for _, path := range []string{upperLogrusPath, lowerLogrusPath} { - if w := getGo(h, path); w.Code != http.StatusOK || w.Body.String() != "zip of "+path { - t.Fatalf("GET %s: status = %d, body = %q", path, w.Code, w.Body) - } - } - } - current, err := proxy.DB.GetArtifact("pkg:golang/github.com/Sirupsen/logrus@v1.0.6", old.Filename) - if err != nil || current == nil { - t.Fatalf("moved artifact = %+v, %v", current, err) - } - if current.ID != old.ID || current.StoragePath != old.StoragePath || current.FetchedAt != old.FetchedAt || current.HitCount != old.HitCount+2 { - t.Errorf("artifact history changed: before=%+v, after=%+v", old, current) - } - legacy, err := proxy.DB.GetArtifact(old.VersionPURL, old.Filename) - if err != nil || legacy != nil { - t.Fatalf("legacy artifact = %+v, %v", legacy, err) - } - for path, count := range counts { - if count.Load() != 0 { - t.Errorf("unexpected upstream request: %s", path) - } - } -} - -func TestGoLegacyCacheRequiresExactModuleOrigin(t *testing.T) { - for _, origin := range []string{"", "https://proxy.example" + lowerLogrusPath} { - t.Run(origin, func(t *testing.T) { - upstream, counts := goCaseUpstream(t) - h, proxy := newGoCaseHandler(t, upstream) - seedLegacyUpperGoZip(t, proxy, origin, "another module") - if w := getGo(h, upperLogrusPath); w.Code != http.StatusOK || w.Body.String() != "zip of "+upperLogrusPath { - t.Fatalf("status = %d, body = %q", w.Code, w.Body) - } - if counts[upperLogrusPath].Load() != 1 { - t.Fatal("archive with unproven identity was reused") - } - }) - } -} - -func seedLegacyUpperGoZip(t *testing.T, proxy *Proxy, upstreamURL, content string) *database.Artifact { - t.Helper() - const packagePURL = "pkg:golang/github.com/sirupsen/logrus" - const versionPURL = packagePURL + "@v1.0.6" - const storagePath = "golang/logrus/v1.0.6/legacy/upper.zip" - filename := goZipFilename("github.com/Sirupsen/logrus", "v1.0.6") - if _, _, err := proxy.Storage.Store(context.Background(), storagePath, strings.NewReader(content)); err != nil { - t.Fatal(err) - } - artifact := testArtifact(content, versionPURL, filename, "application/zip") - if err := proxy.updateCacheDB("golang", "github.com/Sirupsen/logrus", packagePURL, upstreamURL, storagePath, artifact, ""); err != nil { - t.Fatal(err) - } - old, err := proxy.DB.GetArtifact(versionPURL, filename) - if err != nil || old == nil { - t.Fatalf("seeded artifact = %+v, %v", old, err) - } - return old -} - -// seedGoZip caches content as github.com/sirupsen/logrus v1.0.6 under the -// plain filename, recorded as fetched from upstreamURL. -func seedGoZip(t *testing.T, proxy *Proxy, upstreamURL, content string) { - t.Helper() - const storagePath = "golang/logrus/v1.0.6/seed/logrus@v1.0.6.zip" - store := proxy.Storage.(*mockStorage) - if _, _, err := store.Store(context.Background(), storagePath, strings.NewReader(content)); err != nil { - t.Fatalf("seeding storage: %v", err) - } - pkgPURL, versionPURL, err := packagePURLStrings("golang", "github.com/sirupsen/logrus", "v1.0.6") - if err != nil { - t.Fatalf("packagePURLStrings: %v", err) - } - artifact := testArtifact(content, versionPURL, "logrus@v1.0.6.zip", "application/zip") - if err := proxy.updateCacheDB("golang", "github.com/sirupsen/logrus", pkgPURL, upstreamURL, storagePath, artifact, ""); err != nil { - t.Fatalf("seeding cache record: %v", err) - } -} diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go deleted file mode 100644 index efcc1e1..0000000 --- a/internal/handler/go_cooldown.go +++ /dev/null @@ -1,289 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "strings" - "time" - - gomodule "golang.org/x/mod/module" - "golang.org/x/mod/semver" - "golang.org/x/sync/errgroup" -) - -// Go cooldown measures a version by the Time in its .info, which the module -// proxy takes from the commit the version points at. That is when the code -// was committed, not when the version was tagged or first fetched. -// -// Versions in cooldown are left out of @v/list and @latest, and their .info, -// .mod and .zip are answered with 403. The go command falls back to the next -// GOPROXY entry on 404 and 410, so with GOPROXY=proxy,direct a 404 would let -// it fetch the version directly. - -// goInfoFetchLimit bounds the .info requests made at once to filter a list. -const goInfoFetchLimit = 8 - -const errGoCooldown = "version is in cooldown" - -// goInfo is the JSON served for a version's .info and for @latest. -type goInfo struct { - Version string `json:"Version"` - Time time.Time `json:"Time"` -} - -func (h *GoHandler) cooldownEnabled() bool { - return h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() -} - -func (h *GoHandler) allowedAt(module string, published time.Time) bool { - if published.IsZero() { - return true - } - return h.proxy.cooldown().IsAllowed("golang", canonicalPackagePURL("golang", module), published) -} - -// writeGoMetadataError answers a failed upstream metadata fetch. -func (h *GoHandler) writeGoMetadataError(w http.ResponseWriter, err error) { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("go metadata fetch failed", "error", err) - http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) -} - -// fetchGoMetadata reads a module's @v/list, @latest, or a version's .info. -// The .info of a canonical version comes from the cache whatever its age, -// like proxyVersionFile serves it. -func (h *GoHandler) fetchGoMetadata(ctx context.Context, escapedModule, module, rest string) ([]byte, error) { - cacheKey, upstreamURL := module+"/"+rest, h.upstreamURL+"/"+escapedModule+"/"+rest - if v, ok := strings.CutPrefix(rest, "@v/"); ok && strings.HasSuffix(v, ".info") && goVersionImmutable(strings.TrimSuffix(v, ".info")) { - res := h.proxy.fetchImmutableMetadata(ctx, "golang", cacheKey, upstreamURL, "*/*") - return res.body, res.err - } - body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "golang", cacheKey, upstreamURL, "*/*") - return body, err -} - -// Keep escaped module paths in publish-time keys for cache compatibility. -// Only canonical versions are stored; branch queries can resolve differently. -func (h *GoHandler) goTimeKey(module, version string) (versionPURL, packagePURL string, ok bool) { - if h.proxy.DB == nil || gomodule.CanonicalVersion(version) != version { - return "", "", false - } - escaped, err := gomodule.EscapePath(module) - if err != nil { - return "", "", false - } - return canonicalVersionPURL("golang", escaped, version), canonicalPackagePURL("golang", escaped), true -} - -// publishTime returns a version's time from the versions table, or from its -// .info, which is then stored. A version upstream doesn't know, or whose -// .info has no time, gets the zero time and is let through, as in the other -// ecosystems. A failed fetch is an error, so a flaky or rate-limited -// upstream can't let a version through. -func (h *GoHandler) publishTime(ctx context.Context, escapedModule, module, version string) (time.Time, error) { - if versionPURL, _, ok := h.goTimeKey(module, version); ok { - if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return ver.PublishedAt.Time, nil - } - } - - escapedVersion, err := gomodule.EscapeVersion(version) - if err != nil { - return time.Time{}, nil - } - body, err := h.fetchGoMetadata(ctx, escapedModule, module, "@v/"+escapedVersion+".info") - if errors.Is(err, ErrUpstreamNotFound) { - return time.Time{}, nil - } - if err != nil { - return time.Time{}, err - } - var info goInfo - if err := json.Unmarshal(body, &info); err != nil { - return time.Time{}, err - } - if !info.Time.IsZero() { - h.storePublishTime(module, version, info.Time) - } - return info.Time, nil -} - -func (h *GoHandler) storePublishTime(module, version string, published time.Time) { - versionPURL, packagePURL, ok := h.goTimeKey(module, version) - if !ok { - return - } - if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, published); err != nil { - h.proxy.Logger.Warn("cooldown: could not store go publish time", - "module", module, "version", version, "error", err) - } -} - -// checkCooldown answers a request for an exact version in cooldown, or whose -// time couldn't be read, and reports whether it did. -func (h *GoHandler) checkCooldown(w http.ResponseWriter, r *http.Request, escapedModule, module, version string) bool { - if !h.cooldownEnabled() { - return false - } - published, err := h.publishTime(r.Context(), escapedModule, module, version) - if err != nil { - h.proxy.Logger.Warn("cooldown: could not read go version time", - "module", module, "version", version, "error", err) - http.Error(w, "failed to check cooldown", http.StatusBadGateway) - return true - } - if !h.allowedAt(module, published) { - h.refuseCooldown(w, module, version) - return true - } - return false -} - -func (h *GoHandler) refuseCooldown(w http.ResponseWriter, module, version string) { - h.proxy.Logger.Info("cooldown: withholding go module version", "module", module, "version", version) - http.Error(w, errGoCooldown+": "+canonicalVersionPURL("golang", module, version), http.StatusForbidden) -} - -// allowedVersions returns the listed versions of module that are out of -// cooldown, in their upstream order. -func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module string) ([]string, error) { - body, err := h.fetchGoMetadata(ctx, escapedModule, module, "@v/list") - if err != nil { - return nil, err - } - var versions []string - for line := range strings.SplitSeq(string(body), "\n") { - if v := strings.TrimSpace(line); v != "" { - versions = append(versions, v) - } - } - - keep := make([]bool, len(versions)) - g, gctx := errgroup.WithContext(ctx) - g.SetLimit(goInfoFetchLimit) - for i, v := range versions { - g.Go(func() error { - published, err := h.publishTime(gctx, escapedModule, module, v) - if err != nil { - return fmt.Errorf("reading time of %s: %w", v, err) - } - keep[i] = h.allowedAt(module, published) - return nil - }) - } - if err := g.Wait(); err != nil { - return nil, err - } - - allowed := versions[:0] - for i, v := range versions { - if keep[i] { - allowed = append(allowed, v) - } else { - h.proxy.Logger.Info("cooldown: filtering go module version", "module", module, "version", v) - } - } - return allowed, nil -} - -// serveCooldownList serves @v/list without versions in cooldown. -func (h *GoHandler) serveCooldownList(w http.ResponseWriter, r *http.Request, escapedModule, module string) { - allowed, err := h.allowedVersions(r.Context(), escapedModule, module) - if err != nil { - h.writeGoMetadataError(w, err) - return - } - w.Header().Set(headerContentType, "text/plain; charset=utf-8") - if len(allowed) > 0 { - _, _ = w.Write([]byte(strings.Join(allowed, "\n") + "\n")) - } -} - -// serveCooldownInfo serves a .info, refusing it when the version it names is -// in cooldown. rest is the request after /@v/: an escaped version or a query -// such as a branch name, followed by .info. -func (h *GoHandler) serveCooldownInfo(w http.ResponseWriter, r *http.Request, escapedModule, module, rest string) { - body, err := h.fetchGoMetadata(r.Context(), escapedModule, module, "@v/"+rest) - if err != nil { - h.writeGoMetadataError(w, err) - return - } - var info goInfo - if err := json.Unmarshal(body, &info); err == nil && info.Version != "" { - if !h.allowedAt(module, info.Time) { - h.refuseCooldown(w, module, info.Version) - return - } - if !info.Time.IsZero() { - h.storePublishTime(module, info.Version, info.Time) - } - } - w.Header().Set(headerContentType, contentTypeJSON) - _, _ = w.Write(body) -} - -// serveCooldownLatest serves @latest. When upstream's answer is in cooldown, -// it serves the newest listed version out of cooldown instead: the highest -// release, or else the highest pre-release, which is how the go command -// picks from a list. -func (h *GoHandler) serveCooldownLatest(w http.ResponseWriter, r *http.Request, escapedModule, module string) { - body, err := h.fetchGoMetadata(r.Context(), escapedModule, module, "@latest") - if err != nil { - h.writeGoMetadataError(w, err) - return - } - var info goInfo - if err := json.Unmarshal(body, &info); err != nil || h.allowedAt(module, info.Time) { - w.Header().Set(headerContentType, contentTypeJSON) - _, _ = w.Write(body) - return - } - - allowed, err := h.allowedVersions(r.Context(), escapedModule, module) - if err != nil { - h.writeGoMetadataError(w, err) - return - } - best := newestGoVersion(allowed) - if best == "" { - h.refuseCooldown(w, module, info.Version) - return - } - escapedVersion, err := gomodule.EscapeVersion(best) - if err != nil { - h.refuseCooldown(w, module, info.Version) - return - } - h.serveCooldownInfo(w, r, escapedModule, module, escapedVersion+".info") -} - -// newestGoVersion returns the highest release in versions, or the highest -// pre-release when there is no release. -func newestGoVersion(versions []string) string { - var release, prerelease string - for _, v := range versions { - if !semver.IsValid(v) { - continue - } - if semver.Prerelease(v) == "" { - if release == "" || semver.Compare(v, release) > 0 { - release = v - } - } else if prerelease == "" || semver.Compare(v, prerelease) > 0 { - prerelease = v - } - } - if release != "" { - return release - } - return prerelease -} diff --git a/internal/handler/go_cooldown_test.go b/internal/handler/go_cooldown_test.go deleted file mode 100644 index bedc334..0000000 --- a/internal/handler/go_cooldown_test.go +++ /dev/null @@ -1,260 +0,0 @@ -package handler - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -type goCooldownUpstream struct { - server *httptest.Server - mu sync.Mutex - hits map[string]int -} - -func (u *goCooldownUpstream) count(path string) int { - u.mu.Lock() - defer u.mu.Unlock() - return u.hits[path] -} - -// newGoCooldownUpstream serves example.com/mod with an old v1.0.0, a -// recent v1.1.0 and a recent v1.2.0-rc.1, and @latest pointing at v1.1.0. -func newGoCooldownUpstream(t *testing.T, latest string) *goCooldownUpstream { - t.Helper() - now := time.Now() - times := map[string]time.Time{ - "v1.0.0": now.Add(-10 * 24 * time.Hour), - "v1.1.0": now.Add(-time.Hour), - "v1.2.0-rc.1": now.Add(-2 * time.Hour), - } - u := &goCooldownUpstream{hits: map[string]int{}} - u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - u.mu.Lock() - u.hits[r.URL.Path]++ - u.mu.Unlock() - rest, ok := strings.CutPrefix(r.URL.Path, "/example.com/mod/") - if !ok { - http.NotFound(w, r) - return - } - writeInfo := func(v string) { - _ = json.NewEncoder(w).Encode(goInfo{Version: v, Time: times[v]}) - } - switch { - case rest == "@v/list": - _, _ = w.Write([]byte("v1.0.0\nv1.1.0\nv1.2.0-rc.1\n")) - case rest == "@latest": - writeInfo(latest) - case rest == "@v/main.info": - writeInfo("v1.1.0") - case strings.HasSuffix(rest, ".info"): - v := strings.TrimSuffix(strings.TrimPrefix(rest, "@v/"), ".info") - if _, ok := times[v]; !ok { - http.NotFound(w, r) - return - } - writeInfo(v) - case strings.HasSuffix(rest, ".mod"): - _, _ = w.Write([]byte("module example.com/mod\n")) - default: - http.NotFound(w, r) - } - })) - t.Cleanup(u.server.Close) - return u -} - -func goCooldownHandler(t *testing.T, u *goCooldownUpstream) (http.Handler, *mockFetcher, *Proxy) { - t.Helper() - proxy, _, _, fetcher := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - return NewGoHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes(), fetcher, proxy -} - -func goGet(h http.Handler, path string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - return w -} - -func TestGoCooldownFiltersList(t *testing.T) { - u := newGoCooldownUpstream(t, "v1.1.0") - h, _, _ := goCooldownHandler(t, u) - - w := goGet(h, "/example.com/mod/@v/list") - if w.Code != http.StatusOK { - t.Fatalf("status = %d", w.Code) - } - if got := w.Body.String(); got != "v1.0.0\n" { - t.Fatalf("list = %q, want only v1.0.0", got) - } - - // Publish times are stored, so a second list needs no .info requests. - _ = goGet(h, "/example.com/mod/@v/list") - if n := u.count("/example.com/mod/@v/v1.0.0.info"); n != 1 { - t.Errorf("v1.0.0.info fetched %d times, want 1", n) - } -} - -func TestGoCooldownRefusesVersions(t *testing.T) { - u := newGoCooldownUpstream(t, "v1.1.0") - h, fetcher, _ := goCooldownHandler(t, u) - - for _, p := range []string{ - "/example.com/mod/@v/v1.1.0.info", - "/example.com/mod/@v/v1.1.0.mod", - "/example.com/mod/@v/v1.1.0.zip", - "/example.com/mod/@v/main.info", - } { - w := goGet(h, p) - if w.Code != http.StatusForbidden || !strings.Contains(w.Body.String(), errGoCooldown) { - t.Errorf("%s: status = %d body %q, want 403 cooldown", p, w.Code, w.Body.String()) - } - } - if fetcher.fetchCalled { - t.Error("zip in cooldown was fetched") - } - - for _, p := range []string{"/example.com/mod/@v/v1.0.0.info", "/example.com/mod/@v/v1.0.0.mod"} { - if w := goGet(h, p); w.Code != http.StatusOK { - t.Errorf("%s: status = %d, want 200", p, w.Code) - } - } -} - -func TestGoCooldownLatest(t *testing.T) { - u := newGoCooldownUpstream(t, "v1.1.0") - h, _, _ := goCooldownHandler(t, u) - - w := goGet(h, "/example.com/mod/@latest") - if w.Code != http.StatusOK { - t.Fatalf("status = %d: %s", w.Code, w.Body) - } - var info goInfo - if err := json.Unmarshal(w.Body.Bytes(), &info); err != nil { - t.Fatal(err) - } - if info.Version != "v1.0.0" { - t.Fatalf("@latest = %s, want v1.0.0", info.Version) - } -} - -func TestGoCooldownLatestAllowed(t *testing.T) { - u := newGoCooldownUpstream(t, "v1.0.0") - h, _, _ := goCooldownHandler(t, u) - - w := goGet(h, "/example.com/mod/@latest") - var info goInfo - if err := json.Unmarshal(w.Body.Bytes(), &info); err != nil || info.Version != "v1.0.0" { - t.Fatalf("@latest = %d %s, want v1.0.0", w.Code, w.Body) - } - if n := u.count("/example.com/mod/@v/list"); n != 0 { - t.Errorf("list fetched %d times for an allowed @latest", n) - } -} - -func TestGoCooldownDisabledPassesThrough(t *testing.T) { - u := newGoCooldownUpstream(t, "v1.1.0") - h, _, proxy := goCooldownHandler(t, u) - proxy.Cooldown = nil - - if w := goGet(h, "/example.com/mod/@v/list"); w.Body.String() != "v1.0.0\nv1.1.0\nv1.2.0-rc.1\n" { - t.Fatalf("list = %q", w.Body.String()) - } - if w := goGet(h, "/example.com/mod/@v/v1.1.0.info"); w.Code != http.StatusOK { - t.Fatalf("info status = %d", w.Code) - } -} - -func TestNewestGoVersion(t *testing.T) { - tests := []struct { - versions []string - want string - }{ - {[]string{"v1.0.0", "v1.10.0", "v1.9.0", "v2.0.0-rc.1"}, "v1.10.0"}, - {[]string{"v1.0.0-alpha", "v1.0.0-beta"}, "v1.0.0-beta"}, - {[]string{"v2.0.0+incompatible", "v1.5.0"}, "v2.0.0+incompatible"}, - {nil, ""}, - } - for _, tt := range tests { - if got := newestGoVersion(tt.versions); got != tt.want { - t.Errorf("newestGoVersion(%v) = %q, want %q", tt.versions, got, tt.want) - } - } -} - -func TestGoCooldownFailsClosedOnUpstreamErrors(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case strings.HasSuffix(r.URL.Path, "/@v/list"): - _, _ = w.Write([]byte("v1.0.0\n")) - case strings.HasSuffix(r.URL.Path, ".info"): - http.Error(w, "slow down", http.StatusTooManyRequests) - default: - _, _ = w.Write([]byte("module example.com/mod\n")) - } - })) - defer upstream.Close() - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", upstream.URL).Routes() - - for _, p := range []string{ - "/example.com/mod/@v/list", - "/example.com/mod/@v/v1.0.0.mod", - "/example.com/mod/@v/v1.0.0.zip", - } { - if w := goGet(h, p); w.Code != http.StatusBadGateway { - t.Errorf("%s: status = %d, want 502", p, w.Code) - } - } - if fetcher.fetchCalled { - t.Error("zip was fetched without a known time") - } -} - -func TestGoCooldownStoresTimesPerExactModule(t *testing.T) { - u := newGoCooldownUpstream(t, "v1.1.0") - proxy, db, _, _ := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes() - - _ = goGet(h, "/example.com/mod/@v/list") - ver, err := db.GetVersionByPURL(canonicalVersionPURL("golang", "example.com/mod", "v1.0.0")) - if err != nil || ver == nil || !ver.PublishedAt.Valid { - t.Fatalf("time of v1.0.0 not stored: %v %v", ver, err) - } - - gh := &GoHandler{proxy: proxy} - upper, _, ok := gh.goTimeKey("github.com/Azure/sdk", "v1.0.0") - lower, _, _ := gh.goTimeKey("github.com/azure/sdk", "v1.0.0") - if !ok || upper == lower { - t.Errorf("modules differing in case share a key: %q %q", upper, lower) - } - for _, query := range []string{"master", "v1.2", "v1.0.0+meta"} { - if _, _, ok := gh.goTimeKey("example.com/mod", query); ok { - t.Errorf("time of non-canonical %q would be stored", query) - } - } -} - -func TestGoRejectsBadlyEscapedVersions(t *testing.T) { - u := newGoCooldownUpstream(t, "v1.1.0") - h, fetcher, _ := goCooldownHandler(t, u) - for _, p := range []string{"/example.com/mod/@v/V1.0.0.zip", "/example.com/mod/@v/V1.0.0.mod"} { - if w := goGet(h, p); w.Code != http.StatusNotFound { - t.Errorf("%s: status = %d, want 404", p, w.Code) - } - } - if fetcher.fetchCalled || u.count("/example.com/mod/@v/V1.0.0.mod") != 0 { - t.Error("badly escaped version reached upstream") - } -} diff --git a/internal/handler/go_policy_test.go b/internal/handler/go_policy_test.go deleted file mode 100644 index 0ce981f..0000000 --- a/internal/handler/go_policy_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package handler - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/proxy/internal/config" -) - -func TestGoCooldownOverridesIgnoreModuleCase(t *testing.T) { - for _, tc := range []struct { - name, rule, fallback, override string - status int - }{ - {"lowercase restriction", "example.com/module", "0", "7d", http.StatusForbidden}, - {"mixed-case restriction", "example.com/Module", "0", "7d", http.StatusForbidden}, - {"lowercase exemption", "example.com/module", "7d", "0", http.StatusOK}, - {"mixed-case exemption", "example.com/Module", "7d", "0", http.StatusOK}, - } { - t.Run(tc.name, func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasSuffix(r.URL.Path, ".info") { - _ = json.NewEncoder(w).Encode(goInfo{Version: "v1.0.0", Time: time.Now()}) - return - } - _, _ = w.Write([]byte("module zip")) - })) - defer upstream.Close() - h, proxy := newGoCaseHandler(t, upstream) - cfg := config.CooldownConfig{Packages: map[string]string{"pkg:golang/" + tc.rule: tc.override}} - proxy.Cooldown = &cooldown.Config{Default: tc.fallback, Packages: cfg.NormalizedPackages()} - for _, path := range []string{ - "/example.com/!module/@v/v1.0.0.zip", "/example.com/module/@v/v1.0.0.zip", - "/example.com/!module/@v/v1.0.0.info", "/example.com/module/@v/v1.0.0.info", - } { - if response := getGo(h, path); response.Code != tc.status { - t.Errorf("GET %s = %d %s, want %d", path, response.Code, response.Body, tc.status) - } - } - }) - } -} diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go deleted file mode 100644 index 8b38206..0000000 --- a/internal/handler/go_sumdb.go +++ /dev/null @@ -1,432 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "net/http" - "strconv" - "strings" - "sync" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/metrics" - "golang.org/x/mod/module" - "golang.org/x/mod/semver" - "golang.org/x/mod/sumdb/tlog" -) - -const ( - // defaultGoSumDBName is the checksum database the go command uses when - // GOSUMDB is not set. - defaultGoSumDBName = "sum.golang.org" - defaultGoSumDBURL = "https://sum.golang.org" - goEcosystem = "golang" - - // goSumDBProbeBackoff is how long supported keeps answering 404 after the - // proxy failed to reach a checksum database, before it tries again. - goSumDBProbeBackoff = time.Minute - // goSumDBProbeTimeout is how long supported waits for the database. The - // go command's own request has no timeout, so a database that drops - // packets must not hold every build for the full upstream timeout. - goSumDBProbeTimeout = 5 * time.Second -) - -// goSumDBProbes remembers when the proxy last failed to reach each checksum -// database, so a database that is unreachable does not make every go command -// wait for an upstream timeout before it falls back. -type goSumDBProbes struct { - // timeout overrides goSumDBProbeTimeout when set (tests). - timeout time.Duration - - mu sync.Mutex - failed map[string]time.Time - succeeded map[string]time.Time -} - -func (p *goSumDBProbes) probeTimeout() time.Duration { - if p.timeout > 0 { - return p.timeout - } - return goSumDBProbeTimeout -} - -func (p *goSumDBProbes) recentlyFailed(name string) bool { - p.mu.Lock() - defer p.mu.Unlock() - failedAt, ok := p.failed[name] - return ok && time.Since(failedAt) < goSumDBProbeBackoff -} - -func (p *goSumDBProbes) record(name string, reachable bool) { - p.mu.Lock() - defer p.mu.Unlock() - if reachable { - delete(p.failed, name) - if p.succeeded == nil { - p.succeeded = make(map[string]time.Time) - } - p.succeeded[name] = time.Now() - return - } - if p.failed == nil { - p.failed = make(map[string]time.Time) - } - p.failed[name] = time.Now() -} - -// recordFailure records that the database did not answer a request started -// at start, unless a probe has reached it since: a probe that timed out, or -// a lookup that fell back to the cache, must not overwrite a newer success. -func (p *goSumDBProbes) recordFailure(name string, start time.Time) { - p.mu.Lock() - defer p.mu.Unlock() - if !p.succeeded[name].Before(start) { - return - } - if p.failed == nil { - p.failed = make(map[string]time.Time) - } - p.failed[name] = time.Now() -} - -// SetSumDBs sets the checksum databases served at /sumdb/{name}/, mapping -// each database name to its base URL. An empty map keeps the default, -// sum.golang.org. offline makes a cached tree of any age enough to keep -// answering for a database the proxy cannot reach, for networks where -// clients have no route to the database of their own. -func (h *GoHandler) SetSumDBs(sumDBs map[string]string, offline bool) { - h.sumDBOffline = offline - if len(sumDBs) == 0 { - h.sumDBs = map[string]string{defaultGoSumDBName: defaultGoSumDBURL} - return - } - h.sumDBs = make(map[string]string, len(sumDBs)) - for name, sumDBURL := range sumDBs { - h.sumDBs[name] = strings.TrimSuffix(sumDBURL, "/") - } -} - -// handleSumDB serves checksum database requests that the go command sends -// through GOPROXY as /sumdb/{name}/{path}. The module proxy upstream does not -// serve them, so they are fetched from the configured database itself. -// Unknown names get 404, which makes the go command contact the database -// directly. -func (h *GoHandler) handleSumDB(w http.ResponseWriter, r *http.Request, rest string) { - name, path, _ := strings.Cut(rest, "/") - sumDBURL, ok := h.sumDBs[name] - if !ok { - http.NotFound(w, r) - return - } - if path == "supported" { - if h.sumDBReachable(r, name, sumDBURL) { - w.WriteHeader(http.StatusOK) - } else { - http.NotFound(w, r) - } - return - } - immutable, valid := parseGoSumDBPath(path) - if !valid { - http.NotFound(w, r) - return - } - // Only mirrors and auditors fetch data tiles, never the go command, and - // together they hold the whole log, so they are not stored. - if tile, err := tlog.ParseTilePath(path); err == nil && tile.L < 0 { - h.proxy.proxyMetadataStream(w, r, sumDBURL+"/"+path, "identity", "*/*") - return - } - - cacheKey := goSumDBCacheKey(name, sumDBURL, path) - check := goSumDBResponseCheck(path) - served, damaged := false, false - if immutable { - served, damaged = h.serveCachedSumDBTile(w, r, cacheKey, check) - } - if served { - return - } - h.proxySumDB(w, r, name, sumDBURL+"/"+path, cacheKey, check, damaged) -} - -// sumDBReachable reports whether the proxy can serve a checksum database: it -// can fetch the latest signed tree, or has it cached. Once supported answers -// 200 the go command uses only the proxy for that database and never falls -// back to contacting it directly, so a database the proxy cannot reach gets -// 404 instead. The probe goes through the same cache entry as a latest -// request: with cache_metadata it costs at most one upstream request per -// metadata TTL, without it each probe fetches latest. -func (h *GoHandler) sumDBReachable(r *http.Request, name, sumDBURL string) bool { - cacheKey := goSumDBCacheKey(name, sumDBURL, "latest") - treeCheck := goSumDBResponseCheck("latest") - if h.sumDBProbes.recentlyFailed(name) { - return h.cachedTreeServes(r.Context(), cacheKey, treeCheck) - } - // The fetch is shared with concurrent callers and runs detached from this - // request, so it is bounded by the upstream timeout, not by ours. Wait - // for it only up to the probe timeout; it finishes in the background, - // records its own outcome and fills the cache for the next probe. - start := time.Now() - done := make(chan error, 1) - go func() { - done <- h.probeSumDB(context.WithoutCancel(r.Context()), name, cacheKey, sumDBURL, treeCheck) - }() - var err error - select { - case err = <-done: - case <-time.After(h.sumDBProbes.probeTimeout()): - err = fmt.Errorf("no answer within %s", h.sumDBProbes.probeTimeout()) - h.sumDBProbes.recordFailure(name, start) - case <-r.Context().Done(): - // The client went away; that says nothing about the database. - return false - } - if err == nil { - return true - } - // The stale-cache fallback inside the fetch only runs once upstream has - // failed, which a silent database delays past the probe timeout. A tree - // cached earlier may still serve lookups, so look for it here. - if h.cachedTreeServes(r.Context(), cacheKey, treeCheck) { - return true - } - h.proxy.Logger.Info("checksum database unreachable, sending go clients to it directly", - "name", name, "url", sumDBURL, "error", err) - return false -} - -// probeSumDB fetches the database's latest tree and records whether the -// database answered. A tree still fresh in the cache is enough for supported -// but records nothing, since the database was not asked: counting it as an -// answer would keep lookups that fail during an outage from being -// remembered. A tree served because upstream failed is not an answer. -func (h *GoHandler) probeSumDB(ctx context.Context, name, cacheKey, sumDBURL string, check func([]byte) error) error { - if entry, _, _, ok := h.readCachedSumDB(ctx, cacheKey, check); ok && h.treeFresh(entry) { - return nil - } - res := h.proxy.lookupMetadata(ctx, goEcosystem, cacheKey, sumDBURL+"/latest", "*/*", "identity", check) - err := res.err - if err == nil && res.stale { - err = errors.New("checksum database did not answer, tree served from cache") - } - h.sumDBProbes.record(name, err == nil) - return err -} - -// cachedTreeServes reports whether a cached tree lets the proxy answer for -// the database although the database did not: always with go_sumdbs_offline, -// otherwise only while the tree is fresh. -func (h *GoHandler) cachedTreeServes(ctx context.Context, cacheKey string, check func([]byte) error) bool { - entry, _, _, ok := h.readCachedSumDB(ctx, cacheKey, check) - return ok && (h.sumDBOffline || h.treeFresh(entry)) -} - -// treeFresh reports whether a cached tree is within the metadata TTL. -func (h *GoHandler) treeFresh(entry *database.MetadataCacheEntry) bool { - ttl := h.proxy.MetadataTTL - return ttl > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < ttl -} - -// readCachedSumDB reads a checksum database response from the metadata -// cache regardless of its age. It reports false when nothing usable is -// cached. -func (h *GoHandler) readCachedSumDB(ctx context.Context, cacheKey string, check func([]byte) error) (entry *database.MetadataCacheEntry, body []byte, contentType string, ok bool) { - p := h.proxy - if !p.CacheMetadata || p.DB == nil { - return nil, nil, "", false - } - entry, err := p.DB.GetMetadataCache(goEcosystem, cacheKey) - if err != nil || entry == nil { - return nil, nil, "", false - } - body, contentType, err = p.readCachedMetadata(ctx, entry, check) - if err != nil { - return nil, nil, "", false - } - return entry, body, contentType, true -} - -// parseGoSumDBPath reports whether path is a request the checksum database -// protocol defines and whether its response can never change. Only complete -// tiles are immutable; partial tiles, latest and lookups follow the metadata -// TTL. -func parseGoSumDBPath(path string) (immutable, valid bool) { - if path == "latest" { - return false, true - } - if mod, ok := strings.CutPrefix(path, "lookup/"); ok { - return false, validGoSumDBLookup(mod) - } - tile, err := tlog.ParseTilePath(path) - if err != nil { - return false, false - } - return tile.W == 1< 0: - w.Header().Set(headerContentType, "text/plain; charset=utf-8") - w.Header().Set("X-Content-Type-Options", "nosniff") - w.WriteHeader(http.StatusNotFound) - _, _ = w.Write(notFound.body) - case errors.Is(err, ErrUpstreamNotFound): - http.Error(w, "not found", http.StatusNotFound) - default: - if writeUpstreamHeld(w, err) { - return - } - p.Logger.Error("checksum database fetch failed", "url", upstreamURL, "error", err) - http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) - } - return - } - - p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) -} - -// fetchSumDB fetches a checksum database response through the metadata -// cache. A cached copy that failed check is fetched again without -// revalidating it: past the TTL, a database that sends an ETag would answer -// 304 and hand back the damaged copy for good. -func (h *GoHandler) fetchSumDB(ctx context.Context, upstreamURL, cacheKey string, check func([]byte) error, damaged bool) metadataResult { - p := h.proxy - if damaged { - return p.fetchMetadataFromUpstream(ctx, goEcosystem, cacheKey, upstreamURL, "*/*", "identity", check, nil) - } - return p.lookupMetadata(ctx, goEcosystem, cacheKey, upstreamURL, "*/*", "identity", check) -} diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go deleted file mode 100644 index 6a0c47b..0000000 --- a/internal/handler/go_sumdb_test.go +++ /dev/null @@ -1,987 +0,0 @@ -package handler - -import ( - "bytes" - "crypto/rand" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "golang.org/x/mod/sumdb" - "golang.org/x/mod/sumdb/note" - "golang.org/x/mod/sumdb/tlog" -) - -const ( - testSumDBName = "sum.example" - // testSumDBTree looks like a signed tree as far as the proxy checks. - testSumDBTree = "go.sum database tree\n1\nhash\n\n— sum.example signature\n" - // testSumDBRecord looks like a lookup answer as far as the proxy checks. - testSumDBRecord = "1\ngolang.org/x/text v0.14.0 h1:hash=\ngolang.org/x/text v0.14.0/go.mod h1:hash=\n\n" + testSumDBTree -) - -// hashTile returns a hash tile body of the given width. -func hashTile(width int, fill string) string { - return strings.Repeat(fill, width*tlog.HashSize) -} - -// dataTile returns a data tile body holding width records, laid out the way -// sum.golang.org serves it: a blank line between records, none after the -// last. -func dataTile(width int) string { - records := make([]string, width) - for i := range records { - records[i] = fmt.Sprintf("example.com/m%d v1.0.0 h1:hash=\nexample.com/m%d v1.0.0/go.mod h1:hash=\n", i, i) - } - return strings.Join(records, "\n") -} - -// fakeSumDB is a checksum database upstream serving fixed files. Unknown -// paths get 404 with the body from notFound, or a default text. -type fakeSumDB struct { - server *httptest.Server - - mu sync.Mutex - files map[string]string - notFound map[string]string - hits map[string]int - down atomic.Bool -} - -func newFakeSumDB(t *testing.T, files map[string]string) *fakeSumDB { - t.Helper() - if files == nil { - files = map[string]string{} - } - db := &fakeSumDB{files: files, notFound: map[string]string{}, hits: map[string]int{}} - db.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - db.mu.Lock() - db.hits[r.URL.Path]++ - body, ok := db.files[r.URL.Path] - text, custom := db.notFound[r.URL.Path] - db.mu.Unlock() - if db.down.Load() { - http.Error(w, "unavailable", http.StatusInternalServerError) - return - } - if !ok { - if !custom { - text = "not found: " + r.URL.Path - } - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.WriteHeader(http.StatusNotFound) - _, _ = io.WriteString(w, text) - return - } - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - _, _ = io.WriteString(w, body) - })) - t.Cleanup(db.server.Close) - return db -} - -func (db *fakeSumDB) setFile(path, body string) { - db.mu.Lock() - defer db.mu.Unlock() - db.files[path] = body -} - -func (db *fakeSumDB) setNotFound(path, text string) { - db.mu.Lock() - defer db.mu.Unlock() - db.notFound[path] = text -} - -func (db *fakeSumDB) requests(path string) int { - db.mu.Lock() - defer db.mu.Unlock() - return db.hits[path] -} - -func (db *fakeSumDB) totalRequests() int { - db.mu.Lock() - defer db.mu.Unlock() - total := 0 - for _, n := range db.hits { - total += n - } - return total -} - -func newSumDBTestHandler(t *testing.T, sumDBURL string, cacheMetadata bool, ttl time.Duration) *GoHandler { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - proxy.CacheMetadata = cacheMetadata - proxy.MetadataTTL = ttl - h := NewGoHandler(proxy, "http://proxy.example") - h.SetSumDBs(map[string]string{testSumDBName: sumDBURL}, false) - return h -} - -func serveGoRequest(h *GoHandler, target string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - return w -} - -func TestGoSumDB_DefaultsToSumGolangOrg(t *testing.T) { - h := NewGoHandler(nil, "http://proxy.example") - if got := h.sumDBs[defaultGoSumDBName]; got != defaultGoSumDBURL { - t.Errorf("default checksum database = %q, want %q", got, defaultGoSumDBURL) - } - h.SetSumDBs(nil, false) - if got := h.sumDBs[defaultGoSumDBName]; got != defaultGoSumDBURL || len(h.sumDBs) != 1 { - t.Errorf("SetSumDBs(nil) = %v, want only the default", h.sumDBs) - } -} - -func TestGoSumDB_SupportedProbe(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) - - for range 2 { - for target, want := range map[string]int{ - "/sumdb/sum.example/supported": http.StatusOK, - "/sumdb/sum.golang.org/supported": http.StatusNotFound, - "/sumdb/other.example/supported": http.StatusNotFound, - } { - if w := serveGoRequest(h, target); w.Code != want { - t.Errorf("%s: status = %d, want %d", target, w.Code, want) - } - } - } - if n := db.totalRequests(); n != 1 { - t.Errorf("upstream requests = %d, want 1: the probe fetches latest once, then uses the cache", n) - } - if h.sumDBProbes.recentlyFailed(testSumDBName) { - t.Error("a probe served from the fresh cache was recorded as a failure") - } -} - -// TestGoSumDB_SupportedFallsBackWhenUnreachable covers the go command's only -// way back to the database: once supported answers 200 it never contacts the -// database directly, so a database the proxy cannot reach must get 404. -func TestGoSumDB_SupportedFallsBackWhenUnreachable(t *testing.T) { - for _, cacheMetadata := range []bool{true, false} { - t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) - db.down.Store(true) - h := newSumDBTestHandler(t, db.server.URL, cacheMetadata, time.Hour) - - for i := range 2 { - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { - t.Fatalf("request %d: status = %d, want 404", i, w.Code) - } - } - if n := db.requests("/latest"); n != 1 { - t.Errorf("probes = %d, want 1: a failed probe is not repeated within %s", n, goSumDBProbeBackoff) - } - }) - } -} - -// TestGoSumDB_SupportedWithCachedTreeWhileUnreachable keeps the offline case: -// a proxy that has served the database before can still serve it from cache. -func TestGoSumDB_SupportedWithCachedTreeWhileUnreachable(t *testing.T) { - t.Run("fresh tree", func(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) - if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { - t.Fatalf("latest: status = %d, want 200", w.Code) - } - db.down.Store(true) - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { - t.Errorf("supported with a fresh cached tree: status = %d, want 200", w.Code) - } - }) - - // A stale tree only counts with go_sumdbs_offline. Without it, clients - // that can reach the database themselves should go there. - for _, offline := range []bool{true, false} { - t.Run(fmt.Sprintf("stale tree offline=%v", offline), func(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - h.sumDBOffline = offline - if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { - t.Fatalf("latest: status = %d, want 200", w.Code) - } - db.down.Store(true) - want := http.StatusNotFound - if offline { - want = http.StatusOK - } - for i := range 2 { // second probe hits the remembered failure - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != want { - t.Errorf("probe %d: status = %d, want %d", i, w.Code, want) - } - } - if n := db.requests("/latest"); n != 2 { - t.Errorf("upstream requests = %d, want 2: the failure is remembered", n) - } - }) - } -} - -// TestGoSumDB_SupportedGivesUpOnSilentDatabase: a database that accepts the -// connection but never answers (a firewall dropping packets) must not hold -// the go command, which waits for supported without a timeout of its own. -func TestGoSumDB_SupportedGivesUpOnSilentDatabase(t *testing.T) { - release := make(chan struct{}) - db := newFakeSumDB(t, nil) - t.Cleanup(func() { close(release) }) // runs before the server closes - db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - <-release - }) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) - h.sumDBProbes.timeout = 50 * time.Millisecond - - answered := make(chan *httptest.ResponseRecorder, 1) - go func() { answered <- serveGoRequest(h, "/sumdb/sum.example/supported") }() - select { - case w := <-answered: - if w.Code != http.StatusNotFound { - t.Fatalf("status = %d, want 404", w.Code) - } - case <-time.After(time.Second): - t.Fatal("supported still waiting for the database after a second") - } -} - -// TestGoSumDB_SupportedWithStaleTreeWhileSilent: the fetch only falls back -// to a stale cached tree once upstream has failed, which a silent database -// delays past the probe timeout. The proxy can still serve from cache, so -// supported must say so. -func TestGoSumDB_SupportedWithStaleTreeWhileSilent(t *testing.T) { - release := make(chan struct{}) - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) - t.Cleanup(func() { close(release) }) - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - h.sumDBOffline = true - h.sumDBProbes.timeout = 50 * time.Millisecond - - if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { - t.Fatalf("latest: status = %d, want 200", w.Code) - } - db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - <-release - }) - - for i := range 2 { // second probe hits the remembered failure - answered := make(chan *httptest.ResponseRecorder, 1) - go func() { answered <- serveGoRequest(h, "/sumdb/sum.example/supported") }() - select { - case w := <-answered: - if w.Code != http.StatusOK { - t.Fatalf("probe %d: status = %d, want 200 from the cached tree", i, w.Code) - } - case <-time.After(time.Second): - t.Fatalf("probe %d: supported still waiting after a second", i) - } - } -} - -// TestGoSumDB_SlowProbeSuccessClearsFailure: a database that answers after -// the probe timeout gets a 404 the first time, but once its answer is in -// the cache, supported says yes without waiting for the next backoff. -func TestGoSumDB_SlowProbeSuccessClearsFailure(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) - inner := db.server.Config.Handler - db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(100 * time.Millisecond) - inner.ServeHTTP(w, r) - }) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) - h.sumDBProbes.timeout = 10 * time.Millisecond - - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { - t.Fatalf("first probe: status = %d, want 404", w.Code) - } - deadline := time.Now().Add(5 * time.Second) - for h.sumDBProbes.recentlyFailed(testSumDBName) { - if time.Now().After(deadline) { - t.Fatal("the late fetch never cleared the failure") - } - time.Sleep(10 * time.Millisecond) - } - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { - t.Errorf("after the late answer: status = %d, want 200", w.Code) - } - if n := db.requests("/latest"); n != 1 { - t.Errorf("upstream requests = %d, want 1: the second probe is a cache hit", n) - } -} - -// TestGoSumDB_StaleFallbackIsNotAnAnswer: the fetch reports no error when -// it falls back to the cached tree after the database failed. That must -// not count as the database answering, or the failure would be forgotten -// and every probe would wait for the timeout again. -func TestGoSumDB_StaleFallbackIsNotAnAnswer(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - h.sumDBOffline = true - h.sumDBProbes.timeout = 10 * time.Millisecond - - if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { - t.Fatalf("latest: status = %d, want 200", w.Code) - } - inner := db.server.Config.Handler - db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(100 * time.Millisecond) - inner.ServeHTTP(w, r) - }) - db.down.Store(true) - - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200 from the cached tree", w.Code) - } - time.Sleep(300 * time.Millisecond) // the fetch has failed and fallen back by now - if !h.sumDBProbes.recentlyFailed(testSumDBName) { - t.Error("the stale fallback cleared the remembered failure") - } -} - -// TestGoSumDB_AnswerCountsWhenCacheWriteFails: an answer from the database -// is an answer even when it could not be cached afterwards, whether or not -// a stale tree was cached before. -func TestGoSumDB_AnswerCountsWhenCacheWriteFails(t *testing.T) { - for _, staleTree := range []bool{false, true} { - t.Run(fmt.Sprintf("stale tree cached=%v", staleTree), func(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - if staleTree { - if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { - t.Fatalf("latest: status = %d, want 200", w.Code) - } - } - h.proxy.Storage.(*mockStorage).storeErr = errors.New("disk full") - - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { - t.Errorf("status = %d, want 200: the database answered", w.Code) - } - if h.sumDBProbes.recentlyFailed(testSumDBName) { - t.Error("an answered probe was recorded as a failure") - } - }) - } -} - -// TestGoSumDB_OfflineServesCacheWithoutWaiting: in offline mode, once the -// database is known to be unreachable, stale lookups must come from the -// cache at once instead of waiting for the upstream timeout each time. -func TestGoSumDB_OfflineServesCacheWithoutWaiting(t *testing.T) { - const lookup = "/lookup/golang.org/x/text@v0.14.0" - release := make(chan struct{}) - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree, lookup: testSumDBRecord}) - t.Cleanup(func() { close(release) }) - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - h.sumDBOffline = true - h.sumDBProbes.timeout = 50 * time.Millisecond - - for _, path := range []string{"/latest", lookup} { - if w := serveGoRequest(h, "/sumdb/sum.example"+path); w.Code != http.StatusOK { - t.Fatalf("%s: status = %d, want 200", path, w.Code) - } - } - db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - <-release - }) - - for _, path := range []string{"/supported", "/latest", lookup} { - answered := make(chan *httptest.ResponseRecorder, 1) - go func() { answered <- serveGoRequest(h, "/sumdb/sum.example"+path) }() - select { - case w := <-answered: - if w.Code != http.StatusOK { - t.Errorf("%s: status = %d, want 200 from the cache", path, w.Code) - } - case <-time.After(time.Second): - t.Fatalf("%s: still waiting for the database after a second", path) - } - } -} - -// TestGoSumDB_StaleLookupRemembersFailure: the go command asks supported -// only once per process. A lookup that falls back to the cache must renew -// the remembered failure itself, or offline mode would wait for the upstream -// timeout on every lookup a minute into a long build. -func TestGoSumDB_StaleLookupRemembersFailure(t *testing.T) { - const lookup = "/lookup/golang.org/x/text@v0.14.0" - db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - h.sumDBOffline = true - - if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { - t.Fatalf("online: status = %d, want 200", w.Code) - } - if h.sumDBProbes.recentlyFailed(testSumDBName) { - t.Fatal("an answered lookup was recorded as a failure") - } - db.down.Store(true) - if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { - t.Fatalf("outage: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) - } - if !h.sumDBProbes.recentlyFailed(testSumDBName) { - t.Error("a lookup served from the cache after the database failed did not remember the failure") - } -} - -// TestGoSumDB_LateStaleLookupKeepsNewerSuccess: a lookup sent while the -// database was failing, which falls back to the cache only after a probe has -// reached the database again, must not mark it unreachable. -func TestGoSumDB_LateStaleLookupKeepsNewerSuccess(t *testing.T) { - const lookup = "/lookup/golang.org/x/text@v0.14.0" - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree, lookup: testSumDBRecord}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { - t.Fatalf("lookup: status = %d, want 200", w.Code) - } - - var sentOnce sync.Once - sent := make(chan struct{}) - release := make(chan struct{}) - inner := db.server.Config.Handler - db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == lookup { - sentOnce.Do(func() { close(sent) }) - <-release - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - inner.ServeHTTP(w, r) - }) - done := make(chan *httptest.ResponseRecorder, 1) - go func() { done <- serveGoRequest(h, "/sumdb/sum.example"+lookup) }() - <-sent - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { - t.Fatalf("supported: status = %d, want 200", w.Code) - } - close(release) - if w := <-done; w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { - t.Fatalf("lookup: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) - } - if h.sumDBProbes.recentlyFailed(testSumDBName) { - t.Error("a lookup that fell back after the probe succeeded marked the database unreachable") - } -} - -// TestGoSumDB_ShortcutNeedsOfflineMode: without go_sumdbs_offline, a -// remembered failure does not keep cached lookups from asking the database, -// which may well be back. -func TestGoSumDB_ShortcutNeedsOfflineMode(t *testing.T) { - const lookup = "/lookup/golang.org/x/text@v0.14.0" - db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { - t.Fatalf("lookup: status = %d, want 200", w.Code) - } - - h.sumDBProbes.record(testSumDBName, false) - if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { - t.Fatalf("lookup with a remembered failure: status = %d, want 200", w.Code) - } - if n := db.requests(lookup); n != 2 { - t.Errorf("database requests = %d, want 2: the cached lookup was served without asking", n) - } -} - -// TestGoSumDB_FreshTreeIsNotAnAnswer: a supported check answered from a -// fresh cached tree did not reach the database, so it must not cancel a -// failure that a lookup started earlier reports afterwards. -func TestGoSumDB_FreshTreeIsNotAnAnswer(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { - t.Fatalf("supported: status = %d, want 200", w.Code) - } - - // Coarse clocks (Windows) can give the first probe's success and start - // the same time; start must come after it. - h.sumDBProbes.mu.Lock() - succeeded := h.sumDBProbes.succeeded[testSumDBName] - h.sumDBProbes.mu.Unlock() - start := time.Now() - for !start.After(succeeded) { - start = time.Now() - } - db.down.Store(true) - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { - t.Fatalf("supported with a fresh cached tree: status = %d, want 200", w.Code) - } - h.sumDBProbes.recordFailure(testSumDBName, start) - if !h.sumDBProbes.recentlyFailed(testSumDBName) { - t.Error("a supported check answered from the cache counted as reaching the database") - } -} - -func TestGoSumDBProbes_TimeoutDoesNotOverwriteLaterSuccess(t *testing.T) { - var probes goSumDBProbes - start := time.Now() - probes.record(testSumDBName, true) - probes.recordFailure(testSumDBName, start) - if probes.recentlyFailed(testSumDBName) { - t.Error("a success recorded after the probe started was overwritten by its timeout") - } - probes.recordFailure(testSumDBName, time.Now().Add(time.Second)) - if !probes.recentlyFailed(testSumDBName) { - t.Error("a timeout with no later success was not recorded") - } -} - -// TestGoSumDB_SupportedRefusesBlockPage: egress proxies answer blocked hosts -// with a 200 page. That is not a reachable database, and must not be cached -// as its signed tree either. -func TestGoSumDB_SupportedRefusesBlockPage(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": "access to this site is blocked"}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) - - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { - t.Errorf("supported: status = %d, want 404", w.Code) - } - if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusBadGateway { - t.Errorf("latest: status = %d, want 502 for a block page", w.Code) - } - - // The page was not cached: once the database answers, its tree is served. - db.setFile("/latest", testSumDBTree) - w := serveGoRequest(h, "/sumdb/sum.example/latest") - if w.Code != http.StatusOK || w.Body.String() != testSumDBTree { - t.Errorf("latest after recovery: status = %d, body = %q, want the signed tree", w.Code, w.Body.String()) - } -} - -// TestGoSumDB_LookupRefusesBlockPage: a block page in place of a lookup must -// not be cached, where it would replace the copy served during an outage. -// That includes a page that happens to start like a record. -func TestGoSumDB_LookupRefusesBlockPage(t *testing.T) { - const lookup = "/lookup/golang.org/x/text@v0.14.0" - pages := map[string]string{ - "html": "access to this site is blocked", - "record without tree": "1\ngolang.org/x/text v0.14.0 h1:hash=\n\naccess to this site is blocked", - } - for name, page := range pages { - t.Run(name, func(t *testing.T) { - db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - - if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { - t.Fatalf("lookup: status = %d, want 200", w.Code) - } - db.setFile(lookup, page) - if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { - t.Errorf("behind a block page: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) - } - - // The page did not replace the record: it is still there in an outage. - db.down.Store(true) - if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { - t.Errorf("in an outage: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) - } - }) - } -} - -func TestGoSumDB_RequestsGoToTheDatabase(t *testing.T) { - var moduleUpstreamRequests atomic.Int32 - moduleUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - moduleUpstreamRequests.Add(1) - http.NotFound(w, r) - })) - defer moduleUpstream.Close() - - const lookup = "/lookup/golang.org/x/text@v0.14.0" - db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - h := NewGoHandlerWithUpstream(proxy, "http://proxy.example", moduleUpstream.URL) - h.SetSumDBs(map[string]string{testSumDBName: db.server.URL + "/"}, false) - - w := serveGoRequest(h, "/sumdb/sum.example"+lookup) - if w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { - t.Fatalf("status = %d, body = %q; want 200 with the database's response", w.Code, w.Body.String()) - } - if n := db.requests(lookup); n != 1 { - t.Errorf("database requests for %s = %d, want 1", lookup, n) - } - if n := moduleUpstreamRequests.Load(); n != 0 { - t.Errorf("module upstream requests = %d, want 0", n) - } -} - -func TestGoSumDB_LookupCachedWithinTTL(t *testing.T) { - const lookup = "/lookup/golang.org/x/text@v0.14.0" - db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) - - for i := range 2 { - w := serveGoRequest(h, "/sumdb/sum.example"+lookup) - if w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { - t.Fatalf("request %d: status = %d, body = %q", i, w.Code, w.Body.String()) - } - } - if n := db.requests(lookup); n != 1 { - t.Errorf("database requests = %d, want 1: the second lookup is a cache hit", n) - } -} - -func TestGoSumDB_CompleteTilesSkipRevalidation(t *testing.T) { - files := map[string]string{ - "/tile/8/0/000": hashTile(256, "a"), - "/tile/8/0/001.p/5": hashTile(5, "b"), - "/latest": testSumDBTree, - } - wantRequests := map[string]int{ - "/tile/8/0/000": 1, - "/tile/8/0/001.p/5": 2, - "/latest": 2, - } - db := newFakeSumDB(t, files) - // Every cached entry is past its TTL by the second request. - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - - for range 2 { - for path, want := range files { - w := serveGoRequest(h, "/sumdb/sum.example"+path) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Fatalf("%s: status = %d, body = %q, want %q", path, w.Code, w.Body.String(), want) - } - } - } - for path, want := range wantRequests { - if n := db.requests(path); n != want { - t.Errorf("%s: database requests = %d, want %d", path, n, want) - } - } -} - -// TestGoSumDB_DataTilesAreNotStored: the go command never fetches data -// tiles, and stored they would add up to the whole log. -func TestGoSumDB_DataTilesAreNotStored(t *testing.T) { - files := map[string]string{ - "/tile/8/data/000": dataTile(256), - "/tile/8/data/001.p/5": dataTile(5), - } - db := newFakeSumDB(t, files) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) - - for range 2 { - for path, want := range files { - w := serveGoRequest(h, "/sumdb/sum.example"+path) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Fatalf("%s: status = %d, body length %d, want the %d-byte tile", path, w.Code, w.Body.Len(), len(want)) - } - } - } - for path := range files { - if n := db.requests(path); n != 2 { - t.Errorf("%s: database requests = %d, want 2", path, n) - } - key := goSumDBCacheKey(testSumDBName, db.server.URL, strings.TrimPrefix(path, "/")) - if entry, _ := h.proxy.DB.GetMetadataCache(goEcosystem, key); entry != nil { - t.Errorf("%s: stored in the metadata cache", path) - } - } -} - -// TestGoSumDB_BadTilesAreNotCached guards complete tiles, which are never -// revalidated once cached: a response of the wrong size must not be stored. -func TestGoSumDB_BadTilesAreNotCached(t *testing.T) { - const tile = "/tile/8/0/000" - db := newFakeSumDB(t, map[string]string{ - tile: "please log in", - }) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) - - if w := serveGoRequest(h, "/sumdb/sum.example"+tile); w.Code != http.StatusBadGateway { - t.Fatalf("wrong-size tile: status = %d, want 502", w.Code) - } - - want := hashTile(256, "a") - db.setFile(tile, want) - w := serveGoRequest(h, "/sumdb/sum.example"+tile) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Fatalf("after upstream recovered: status = %d, body length %d, want the %d-byte tile", w.Code, w.Body.Len(), len(want)) - } - if n := db.requests(tile); n != 2 { - t.Errorf("upstream requests = %d, want 2: the bad response was not cached", n) - } -} - -// TestGoSumDB_DamagedCachedTileIsFetchedAgain guards the check on the cached -// copy: a complete tile is never revalidated, so a damaged blob would -// otherwise be served for good. The database sends an ETag, so past the TTL -// a request that asks whether the tile changed would get 304 and the damaged -// copy back. -func TestGoSumDB_DamagedCachedTileIsFetchedAgain(t *testing.T) { - const tile = "/tile/8/0/000" - want := hashTile(256, "a") - for _, ttl := range []time.Duration{time.Hour, time.Nanosecond} { - t.Run(fmt.Sprintf("ttl=%s", ttl), func(t *testing.T) { - db := newFakeSumDB(t, map[string]string{tile: want}) - inner := db.server.Config.Handler - db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("ETag", `"tile"`) - if r.Header.Get("If-None-Match") == `"tile"` { - w.WriteHeader(http.StatusNotModified) - return - } - inner.ServeHTTP(w, r) - }) - h := newSumDBTestHandler(t, db.server.URL, true, ttl) - - if w := serveGoRequest(h, "/sumdb/sum.example"+tile); w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200", w.Code) - } - - store := h.proxy.Storage.(*mockStorage) - blob := metadataStoragePath(goEcosystem, goSumDBCacheKey(testSumDBName, db.server.URL, "tile/8/0/000")) - store.mu.Lock() - if _, ok := store.files[blob]; !ok { - store.mu.Unlock() - t.Fatalf("tile not stored at %s", blob) - } - store.files[blob] = []byte(want[:100]) - store.mu.Unlock() - - w := serveGoRequest(h, "/sumdb/sum.example"+tile) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Fatalf("after damage: status = %d, body length %d, want the %d-byte tile", w.Code, w.Body.Len(), len(want)) - } - if n := db.requests(tile); n != 2 { - t.Errorf("upstream requests = %d, want 2: the damaged copy was fetched again", n) - } - }) - } -} - -func TestGoSumDB_CompleteTilesNotStoredWithoutMetadataCache(t *testing.T) { - const tile = "/tile/8/0/000" - db := newFakeSumDB(t, map[string]string{tile: hashTile(256, "a")}) - h := newSumDBTestHandler(t, db.server.URL, false, time.Hour) - - for range 2 { - if w := serveGoRequest(h, "/sumdb/sum.example"+tile); w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200", w.Code) - } - } - if n := db.requests(tile); n != 2 { - t.Errorf("database requests = %d, want 2 with cache_metadata off", n) - } -} - -func TestGoSumDB_OfflineFallback(t *testing.T) { - files := map[string]string{ - "/lookup/golang.org/x/text@v0.14.0": testSumDBRecord, - "/latest": testSumDBTree, - "/tile/8/0/000": hashTile(256, "a"), - "/tile/8/0/001.p/5": hashTile(5, "b"), - } - db := newFakeSumDB(t, files) - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - - for path := range files { - if w := serveGoRequest(h, "/sumdb/sum.example"+path); w.Code != http.StatusOK { - t.Fatalf("%s online: status = %d, want 200", path, w.Code) - } - } - - db.down.Store(true) - for path, want := range files { - w := serveGoRequest(h, "/sumdb/sum.example"+path) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Errorf("%s offline: status = %d, body = %q, want 200 with %q", path, w.Code, w.Body.String(), want) - } - } -} - -func TestGoSumDB_NotFoundRelaysUpstreamText(t *testing.T) { - const lookup = "/lookup/github.com/google/uuid@v9.9.9" - const reason = "not found: github.com/google/uuid@v9.9.9: invalid version: unknown revision v9.9.9" - - for _, cacheMetadata := range []bool{true, false} { - t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { - db := newFakeSumDB(t, nil) - db.setNotFound(lookup, reason) - h := newSumDBTestHandler(t, db.server.URL, cacheMetadata, time.Hour) - - w := serveGoRequest(h, "/sumdb/sum.example"+lookup) - if w.Code != http.StatusNotFound { - t.Fatalf("status = %d, want 404", w.Code) - } - if got := strings.TrimSpace(w.Body.String()); got != reason { - t.Errorf("body = %q, want the database's text %q", got, reason) - } - }) - } - - t.Run("long text is cut", func(t *testing.T) { - db := newFakeSumDB(t, nil) - db.setNotFound(lookup, strings.Repeat("x", upstreamNotFoundBodyLimit+1000)) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) - - w := serveGoRequest(h, "/sumdb/sum.example"+lookup) - if w.Code != http.StatusNotFound { - t.Fatalf("status = %d, want 404", w.Code) - } - if got := w.Body.Len(); got != upstreamNotFoundBodyLimit { - t.Errorf("body length = %d, want %d", got, upstreamNotFoundBodyLimit) - } - }) -} - -func TestGoSumDB_InvalidPathsNeverReachUpstream(t *testing.T) { - db := newFakeSumDB(t, nil) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) - - for _, target := range []string{ - "/sumdb/sum.example", - "/sumdb/sum.example/", - "/sumdb/sum.example/other", - "/sumdb/sum.example/latest/extra", - "/sumdb/sum.example/lookup/", - "/sumdb/sum.example/lookup/golang.org/x/text", - "/sumdb/sum.example/lookup/golang.org/x/text@", - "/sumdb/sum.example/lookup/golang.org/x/text@master", - "/sumdb/sum.example/lookup/github.com/Google/uuid@v1.6.0", - "/sumdb/sum.example/lookup/../../etc/passwd@v1.0.0", - "/sumdb/sum.example/tile/x/0/000", - "/sumdb/sum.example/tile/8/0/000.p/0", - "/sumdb/sum.example/tile/8/0/000.p/256", - "/sumdb/sum.example/tile/8/0/../../../latest", - "/sumdb/unknown.example/latest", - } { - if w := serveGoRequest(h, target); w.Code != http.StatusNotFound { - t.Errorf("%s: status = %d, want 404", target, w.Code) - } - } - if n := db.totalRequests(); n != 0 { - t.Errorf("upstream requests = %d, want 0", n) - } -} - -// sumDBClientOps lets the x/mod checksum database client read through the -// proxy, keeping its configuration and cache in memory. -type sumDBClientOps struct { - t *testing.T - remote string - key string - - mu sync.Mutex - config map[string][]byte - cache map[string][]byte -} - -func newSumDBClientOps(t *testing.T, remote, key string) *sumDBClientOps { - return &sumDBClientOps{t: t, remote: remote, key: key, config: map[string][]byte{}, cache: map[string][]byte{}} -} - -func (o *sumDBClientOps) ReadRemote(path string) ([]byte, error) { - resp, err := http.Get(o.remote + path) - if err != nil { - return nil, err - } - defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("GET %s: %s: %s", path, resp.Status, body) - } - return body, nil -} - -func (o *sumDBClientOps) ReadConfig(file string) ([]byte, error) { - if file == "key" { - return []byte(o.key), nil - } - o.mu.Lock() - defer o.mu.Unlock() - return o.config[file], nil -} - -func (o *sumDBClientOps) WriteConfig(file string, old, data []byte) error { - o.mu.Lock() - defer o.mu.Unlock() - if !bytes.Equal(o.config[file], old) { - return sumdb.ErrWriteConflict - } - o.config[file] = data - return nil -} - -func (o *sumDBClientOps) ReadCache(file string) ([]byte, error) { - o.mu.Lock() - defer o.mu.Unlock() - data, ok := o.cache[file] - if !ok { - return nil, errors.New("not cached") - } - return data, nil -} - -func (o *sumDBClientOps) WriteCache(file string, data []byte) { - o.mu.Lock() - defer o.mu.Unlock() - o.cache[file] = data -} - -func (o *sumDBClientOps) Log(msg string) { o.t.Log(msg) } - -func (o *sumDBClientOps) SecurityError(msg string) { - o.t.Errorf("checksum database security error: %s", msg) -} - -// TestGoSumDB_ClientRoundTrip runs the go command's checksum database client -// against a real database implementation through the proxy, so signatures, -// record proofs and tile hashes are checked end to end. A second client then -// verifies the same lookup from the proxy's cache with the database down. -func TestGoSumDB_ClientRoundTrip(t *testing.T) { - signerKey, verifierKey, err := note.GenerateKey(rand.Reader, testSumDBName) - if err != nil { - t.Fatal(err) - } - gosum := func(path, version string) ([]byte, error) { - return fmt.Appendf(nil, "%s %s h1:Y29udGVudA==\n%s %s/go.mod h1:Z28ubW9k\n", path, version, path, version), nil - } - database := sumdb.NewServer(sumdb.NewTestServer(signerKey, gosum)) - var down atomic.Bool - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if down.Load() { - http.Error(w, "unavailable", http.StatusInternalServerError) - return - } - database.ServeHTTP(w, r) - })) - defer upstream.Close() - - h := newSumDBTestHandler(t, upstream.URL, true, time.Hour) - proxy := httptest.NewServer(h.Routes()) - defer proxy.Close() - remote := proxy.URL + "/sumdb/" + testSumDBName - - client := sumdb.NewClient(newSumDBClientOps(t, remote, verifierKey)) - for _, m := range [][2]string{{"rsc.io/quote", "v1.5.2"}, {"golang.org/x/text", "v0.3.0"}, {"github.com/Google/UUID", "v1.6.0"}} { - lines, err := client.Lookup(m[0], m[1]) - if err != nil { - t.Fatalf("lookup %s@%s through the proxy: %v", m[0], m[1], err) - } - if want := m[0] + " " + m[1] + " h1:Y29udGVudA=="; len(lines) != 1 || lines[0] != want { - t.Errorf("lookup %s@%s = %q, want [%q]", m[0], m[1], lines, want) - } - } - - down.Store(true) - offline := sumdb.NewClient(newSumDBClientOps(t, remote, verifierKey)) - if _, err := offline.Lookup("rsc.io/quote", "v1.5.2"); err != nil { - t.Fatalf("lookup from the proxy cache with the database down: %v", err) - } -} diff --git a/internal/handler/go_test.go b/internal/handler/go_test.go index ae998c9..da4ea63 100644 --- a/internal/handler/go_test.go +++ b/internal/handler/go_test.go @@ -1,49 +1,9 @@ package handler import ( - "errors" - "net/http" - "net/http/httptest" "testing" - - "github.com/git-pkgs/registries/fetch" ) -func TestGoModuleDownloadUpstreamErrors(t *testing.T) { - tests := []struct { - name string - fetchErr error - wantStatus int - }{ - { - name: "module not found", - fetchErr: fetch.ErrNotFound, - wantStatus: http.StatusNotFound, - }, - { - name: "upstream failure", - fetchErr: errors.New("connection refused"), - wantStatus: http.StatusBadGateway, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.fetchErr = tt.fetchErr - handler := NewGoHandler(proxy, "http://localhost:8080") - - req := httptest.NewRequest(http.MethodGet, "/example.com/mod/@v/v1.0.0.zip", nil) - resp := httptest.NewRecorder() - handler.Routes().ServeHTTP(resp, req) - - if resp.Code != tt.wantStatus { - t.Fatalf("status = %d, want %d", resp.Code, tt.wantStatus) - } - }) - } -} - func TestDecodeGoModule(t *testing.T) { tests := []struct { encoded string diff --git a/internal/handler/go_vulndb.go b/internal/handler/go_vulndb.go deleted file mode 100644 index d3c1598..0000000 --- a/internal/handler/go_vulndb.go +++ /dev/null @@ -1,128 +0,0 @@ -package handler - -import ( - "bytes" - "compress/gzip" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "strings" -) - -const ( - // defaultGoVulnDBURL is the database govulncheck uses without -db. - defaultGoVulnDBURL = "https://vuln.go.dev" - - // goVulnDBMaxIDLength bounds an entry ID in a request path. Go IDs look - // like GO-2024-1234; other OSV databases use longer ones such as - // GHSA-xxxx-xxxx-xxxx. - goVulnDBMaxIDLength = 128 - - // goVulnDBMaxJSONSize bounds a decompressed .json.gz response while it is - // checked. index/modules.json, the largest file, is a few MB. - goVulnDBMaxJSONSize = 256 << 20 -) - -// SetVulnDB sets the Go vulnerability database served at /vulndb/. An empty -// URL keeps the default, vuln.go.dev. -func (h *GoHandler) SetVulnDB(vulnDBURL string) { - h.vulnDBURL = configuredUpstreamURL(vulnDBURL, defaultGoVulnDBURL) -} - -// handleVulnDB serves the Go vulnerability database at -// /vulndb/{path}. Only the files the database API defines are passed on: -// index/db.json, index/modules.json, index/vulns.json and ID/{id}.json, each -// also as .json.gz, which is what govulncheck fetches. Every file can change, -// so all of them follow the metadata TTL and fall back to the cached copy -// when the database cannot be reached. The bytes are stored and served -// unchanged: a .json.gz file is gzip data, not a gzip-encoded response. -func (h *GoHandler) handleVulnDB(w http.ResponseWriter, r *http.Request, path string) { - if !validGoVulnDBPath(path) { - http.NotFound(w, r) - return - } - upstreamURL := h.vulnDBURL + "/" + path - p := h.proxy - if !p.CacheMetadata { - p.proxyMetadataStream(w, r, upstreamURL, "identity", "*/*") - return - } - cacheKey := goVulnDBCacheKey(h.vulnDBURL, path) - res := p.lookupMetadata(r.Context(), goEcosystem, cacheKey, upstreamURL, "*/*", "identity", goVulnDBResponseCheck(path)) - if res.err != nil { - p.writeMetadataFetchError(w, res.err) - return - } - p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) -} - -// validGoVulnDBPath reports whether path is a file the vulnerability -// database API defines. Entry IDs are limited to letters, digits, '-' and -// '_', which keeps '.', '/', '%' and the like out of the upstream URL. -func validGoVulnDBPath(path string) bool { - name, ok := strings.CutSuffix(path, ".json.gz") - if !ok { - name, ok = strings.CutSuffix(path, ".json") - } - if !ok { - return false - } - switch name { - case "index/db", "index/modules", "index/vulns": - return true - } - id, ok := strings.CutPrefix(name, "ID/") - if !ok || id == "" || len(id) > goVulnDBMaxIDLength { - return false - } - for _, c := range id { - if !isASCIILetterOrDigit(c) && c != '-' && c != '_' { - return false - } - } - return true -} - -func isASCIILetterOrDigit(c rune) bool { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') -} - -// goVulnDBResponseCheck returns a check that a response to path is JSON, -// after decompressing it for a .json.gz path. A 200 that is something else -// (an egress proxy's block page, a cut-off download) is refused rather than -// cached, where it would also replace the copy kept for an outage. -func goVulnDBResponseCheck(path string) func([]byte) error { - compressed := strings.HasSuffix(path, ".gz") - return func(body []byte) error { - data := body - if compressed { - zr, err := gzip.NewReader(bytes.NewReader(body)) - if err != nil { - return fmt.Errorf("vulnerability database file %s is not gzip data: %w", path, err) - } - data, err = io.ReadAll(io.LimitReader(zr, goVulnDBMaxJSONSize+1)) - if err == nil && len(data) > goVulnDBMaxJSONSize { - err = errors.New("too large") - } - if err != nil { - return fmt.Errorf("vulnerability database file %s does not decompress: %w", path, err) - } - } - if !json.Valid(data) { - return fmt.Errorf("vulnerability database file %s is not JSON", path) - } - return nil - } -} - -// goVulnDBCacheKey hashes the database URL and the request path, as -// goSumDBCacheKey does, so repointing upstream.go_vulndb does not serve the -// old database's entries. -func goVulnDBCacheKey(vulnDBURL, path string) string { - digest := sha256.Sum256([]byte(vulnDBURL + "\x00" + path)) - return "vulndb/" + hex.EncodeToString(digest[:]) -} diff --git a/internal/handler/go_vulndb_test.go b/internal/handler/go_vulndb_test.go deleted file mode 100644 index 162ff9a..0000000 --- a/internal/handler/go_vulndb_test.go +++ /dev/null @@ -1,225 +0,0 @@ -package handler - -import ( - "bytes" - "compress/gzip" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" -) - -const ( - testVulnDBDB = `{"modified":"2026-10-21T00:27:29Z"}` - testVulnDBModules = `[{"path":"golang.org/x/text","vulns":[{"id":"GO-2021-0113","modified":"2026-10-21T00:27:29Z","fixed":"0.3.7"}]}]` - testVulnDBEntry = `{"schema_version":"1.3.1","id":"GO-2021-0113","modified":"2026-10-21T00:27:29Z"}` -) - -func gzipString(t *testing.T, s string) string { - t.Helper() - var buf bytes.Buffer - zw := gzip.NewWriter(&buf) - if _, err := io.WriteString(zw, s); err != nil { - t.Fatal(err) - } - if err := zw.Close(); err != nil { - t.Fatal(err) - } - return buf.String() -} - -// testVulnDBFiles is a small vulnerability database laid out as vuln.go.dev -// serves it: every file as .json and .json.gz, the latter as gzip data with a -// JSON content type and no Content-Encoding. -func testVulnDBFiles(t *testing.T) map[string]string { - t.Helper() - files := map[string]string{ - "/index/db.json": testVulnDBDB, - "/index/modules.json": testVulnDBModules, - "/index/vulns.json": `[{"id":"GO-2021-0113","modified":"2026-10-21T00:27:29Z"}]`, - "/ID/GO-2021-0113.json": testVulnDBEntry, - "/ID/GHSA-ppp9-7jff-5vj2.json": testVulnDBEntry, - } - all := make(map[string]string, 2*len(files)) - for path, body := range files { - all[path] = body - all[path+".gz"] = gzipString(t, body) - } - return all -} - -func newVulnDBTestHandler(t *testing.T, vulnDBURL string, cacheMetadata bool, ttl time.Duration) *GoHandler { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - proxy.CacheMetadata = cacheMetadata - proxy.MetadataTTL = ttl - h := NewGoHandler(proxy, "http://proxy.example") - h.SetVulnDB(vulnDBURL) - return h -} - -func TestGoVulnDB_DefaultsToVulnGoDev(t *testing.T) { - h := NewGoHandler(nil, "http://proxy.example") - if h.vulnDBURL != defaultGoVulnDBURL { - t.Errorf("vulnDBURL = %q, want %q", h.vulnDBURL, defaultGoVulnDBURL) - } - h.SetVulnDB("") - if h.vulnDBURL != defaultGoVulnDBURL { - t.Errorf("after SetVulnDB(\"\"): vulnDBURL = %q, want %q", h.vulnDBURL, defaultGoVulnDBURL) - } - h.SetVulnDB("https://vulndb.example/") - if h.vulnDBURL != "https://vulndb.example" { - t.Errorf("vulnDBURL = %q, want the trailing slash trimmed", h.vulnDBURL) - } -} - -func TestGoVulnDB_ServesFilesUnchanged(t *testing.T) { - files := testVulnDBFiles(t) - for _, cacheMetadata := range []bool{true, false} { - db := newFakeSumDB(t, files) - h := newVulnDBTestHandler(t, db.server.URL, cacheMetadata, time.Hour) - for path, want := range files { - w := serveGoRequest(h, "/vulndb"+path) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Errorf("cache_metadata=%v %s: status = %d, body = %q, want 200 with the upstream bytes", cacheMetadata, path, w.Code, w.Body.String()) - } - if enc := w.Header().Get(headerContentEncoding); enc != "" { - t.Errorf("cache_metadata=%v %s: Content-Encoding = %q, want none", cacheMetadata, path, enc) - } - } - } -} - -func TestGoVulnDB_HeadForSchemaCheck(t *testing.T) { - db := newFakeSumDB(t, testVulnDBFiles(t)) - h := newVulnDBTestHandler(t, db.server.URL, true, time.Hour) - - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/vulndb/index/modules.json.gz", nil)) - if w.Code != http.StatusOK { - t.Errorf("HEAD index/modules.json.gz: status = %d, want 200", w.Code) - } - - w = httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/golang.org/x/text/@v/list", nil)) - if w.Code != http.StatusMethodNotAllowed { - t.Errorf("HEAD on a module path: status = %d, want 405", w.Code) - } -} - -func TestGoVulnDB_CachedWithinTTL(t *testing.T) { - db := newFakeSumDB(t, testVulnDBFiles(t)) - h := newVulnDBTestHandler(t, db.server.URL, true, time.Hour) - - for range 3 { - if w := serveGoRequest(h, "/vulndb/index/modules.json.gz"); w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200", w.Code) - } - } - if n := db.requests("/index/modules.json.gz"); n != 1 { - t.Errorf("upstream requests = %d, want 1", n) - } -} - -func TestGoVulnDB_RefreshedAfterTTL(t *testing.T) { - db := newFakeSumDB(t, testVulnDBFiles(t)) - h := newVulnDBTestHandler(t, db.server.URL, true, time.Nanosecond) - - serveGoRequest(h, "/vulndb/ID/GO-2021-0113.json") - updated := `{"id":"GO-2021-0113","modified":"2026-10-27T00:00:00Z"}` - db.setFile("/ID/GO-2021-0113.json", updated) - if w := serveGoRequest(h, "/vulndb/ID/GO-2021-0113.json"); w.Body.String() != updated { - t.Errorf("body = %q, want the modified entry %q", w.Body.String(), updated) - } -} - -func TestGoVulnDB_StaleFallbackWhenUnreachable(t *testing.T) { - files := testVulnDBFiles(t) - db := newFakeSumDB(t, files) - h := newVulnDBTestHandler(t, db.server.URL, true, time.Nanosecond) - - for path := range files { - if w := serveGoRequest(h, "/vulndb"+path); w.Code != http.StatusOK { - t.Fatalf("%s online: status = %d, want 200", path, w.Code) - } - } - - db.down.Store(true) - for path, want := range files { - w := serveGoRequest(h, "/vulndb"+path) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Errorf("%s offline: status = %d, body = %q, want 200 with the cached copy", path, w.Code, w.Body.String()) - } - if w.Header().Get("Warning") == "" { - t.Errorf("%s offline: no Warning header, want the response marked stale", path) - } - } - if w := serveGoRequest(h, "/vulndb/ID/GO-2099-0001.json.gz"); w.Code != http.StatusBadGateway { - t.Errorf("uncached entry offline: status = %d, want 502", w.Code) - } -} - -func TestGoVulnDB_RefusesNonJSON(t *testing.T) { - const blockPage = "blocked by policy" - db := newFakeSumDB(t, map[string]string{ - "/index/db.json": blockPage, - "/index/db.json.gz": blockPage, - "/index/modules.json.gz": gzipString(t, blockPage), - "/ID/GO-2021-0113.json.gz": gzipString(t, testVulnDBEntry)[:20], - }) - h := newVulnDBTestHandler(t, db.server.URL, true, time.Hour) - - for _, path := range []string{"/index/db.json", "/index/db.json.gz", "/index/modules.json.gz", "/ID/GO-2021-0113.json.gz"} { - if w := serveGoRequest(h, "/vulndb"+path); w.Code != http.StatusBadGateway { - t.Errorf("%s: status = %d, want 502", path, w.Code) - } - } - - // Nothing bad was cached: once upstream serves the real file, it is used. - db.setFile("/index/db.json", testVulnDBDB) - if w := serveGoRequest(h, "/vulndb/index/db.json"); w.Code != http.StatusOK || w.Body.String() != testVulnDBDB { - t.Errorf("after fix: status = %d, body = %q, want 200 with %q", w.Code, w.Body.String(), testVulnDBDB) - } -} - -func TestGoVulnDB_NotFound(t *testing.T) { - db := newFakeSumDB(t, nil) - h := newVulnDBTestHandler(t, db.server.URL, true, time.Hour) - if w := serveGoRequest(h, "/vulndb/ID/GO-2099-0001.json.gz"); w.Code != http.StatusNotFound { - t.Errorf("status = %d, want 404", w.Code) - } -} - -func TestGoVulnDB_InvalidPathsNeverReachUpstream(t *testing.T) { - db := newFakeSumDB(t, nil) - h := newVulnDBTestHandler(t, db.server.URL, true, time.Hour) - - for _, target := range []string{ - "/vulndb", - "/vulndb/", - "/vulndb/index", - "/vulndb/index/db", - "/vulndb/index/db.json.zip", - "/vulndb/index/other.json", - "/vulndb/vulndb.zip", - "/vulndb/ID/", - "/vulndb/ID/.json", - "/vulndb/ID/GO-2021-0113", - "/vulndb/ID/GO-2021-0113.txt", - "/vulndb/ID/../index/db.json", - "/vulndb/ID/..%2Findex%2Fdb.json", - "/vulndb/ID/GO.2021.json", - "/vulndb/ID/a/b.json", - "/vulndb/ID/GO%3F.json", - "/vulndb/ID/" + string(bytes.Repeat([]byte("A"), goVulnDBMaxIDLength+1)) + ".json", - } { - if w := serveGoRequest(h, target); w.Code != http.StatusNotFound { - t.Errorf("%s: status = %d, want 404", target, w.Code) - } - } - if n := db.totalRequests(); n != 0 { - t.Errorf("upstream requests = %d, want 0", n) - } -} diff --git a/internal/handler/gradle.go b/internal/handler/gradle.go index cd3e78a..3b703be 100644 --- a/internal/handler/gradle.go +++ b/internal/handler/gradle.go @@ -93,32 +93,32 @@ func (h *GradleBuildCacheHandler) cacheStoragePath(key string) string { func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http.Request, key string) { storagePath := h.cacheStoragePath(key) - w.Header().Set(headerContentType, gradleBuildCacheContentType) + w.Header().Set("Content-Type", gradleBuildCacheContentType) if r.Method == http.MethodHead { existsStart := time.Now() exists, err := h.proxy.Storage.Exists(r.Context(), storagePath) metrics.RecordStorageOperation("read", time.Since(existsStart)) if err != nil { - metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err}) + metrics.RecordStorageError("read") h.proxy.Logger.Error("failed to check gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to read cache entry", http.StatusInternalServerError) return } if !exists { - metrics.RecordArtifactCacheMiss("gradle") + metrics.RecordCacheMiss("gradle") http.NotFound(w, r) return } - metrics.RecordArtifactCacheHit("gradle") + metrics.RecordCacheHit("gradle") sizeStart := time.Now() size, err := h.proxy.Storage.Size(r.Context(), storagePath) metrics.RecordStorageOperation("read", time.Since(sizeStart)) if err != nil { - metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err}) + metrics.RecordStorageError("read") } else if size >= 0 { - w.Header().Set(headerContentLength, strconv.FormatInt(size, 10)) + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) } w.WriteHeader(http.StatusOK) @@ -130,17 +130,17 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http metrics.RecordStorageOperation("read", time.Since(readStart)) if err != nil { if errors.Is(err, storage.ErrNotFound) { - metrics.RecordArtifactCacheMiss("gradle") + metrics.RecordCacheMiss("gradle") http.NotFound(w, r) return } - metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err}) + metrics.RecordStorageError("read") h.proxy.Logger.Error("failed to open gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to read cache entry", http.StatusInternalServerError) return } defer func() { _ = reader.Close() }() - metrics.RecordArtifactCacheHit("gradle") + metrics.RecordCacheHit("gradle") w.WriteHeader(http.StatusOK) _, _ = io.Copy(w, reader) @@ -165,14 +165,14 @@ func (h *GradleBuildCacheHandler) handlePut(w http.ResponseWriter, r *http.Reque return } - metrics.RecordStorageError("write", metrics.FailureDetail{Path: storagePath, Err: err}) + metrics.RecordStorageError("write") h.proxy.Logger.Error("failed to store gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to write cache entry", http.StatusInternalServerError) return } - w.Header().Set(headerContentLength, "0") - w.Header().Set(headerETag, `"`+hash+`"`) + w.Header().Set("Content-Length", "0") + w.Header().Set("ETag", `"`+hash+`"`) w.WriteHeader(http.StatusCreated) } diff --git a/internal/handler/gradle_test.go b/internal/handler/gradle_test.go index a4b95c2..a05d07e 100644 --- a/internal/handler/gradle_test.go +++ b/internal/handler/gradle_test.go @@ -237,8 +237,8 @@ func TestGradleBuildCacheHandler_RecordsMetrics(t *testing.T) { srv := httptest.NewServer(h.Routes()) defer srv.Close() - hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle", metrics.CacheKindArtifact)) - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle", metrics.CacheKindArtifact)) + hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle")) + missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle")) key := "metrics-key" putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader("payload")) @@ -273,8 +273,8 @@ func TestGradleBuildCacheHandler_RecordsMetrics(t *testing.T) { } _ = missResp.Body.Close() - hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle", metrics.CacheKindArtifact)) - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle", metrics.CacheKindArtifact)) + hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle")) + missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle")) if diff := hitsAfter - hitsBefore; diff != 2 { t.Fatalf("cache hits delta = %.0f, want 2", diff) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9612e51..d06ca83 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -13,22 +13,14 @@ import ( "net/url" "strconv" "strings" - "sync" - "sync/atomic" "time" - "github.com/git-pkgs/artifacts" "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/integrity" "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/metrics" - "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/proxy/internal/scanner" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" - "github.com/opencontainers/go-digest" ) // containsPathTraversal returns true if the path contains ".." segments @@ -44,18 +36,6 @@ func containsPathTraversal(path string) bool { return false } -// rejectQueryChars refuses a decoded path containing '?' or '#', which would cut -// the upstream URL short and cache one file under another's name. -func rejectQueryChars(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.ContainsAny(r.URL.Path, "?#") { - http.Error(w, "invalid path", http.StatusBadRequest) - return - } - next.ServeHTTP(w, r) - }) -} - func hasDotDotSegment(path string) bool { path = strings.ReplaceAll(path, "\\", "/") for segment := range strings.SplitSeq(path, "/") { @@ -66,86 +46,15 @@ func hasDotDotSegment(path string) bool { return false } -func configuredUpstreamURL(value, defaultValue string) string { - if value == "" { - value = defaultValue - } - return strings.TrimRight(value, "/") -} - const defaultHTTPTimeout = 30 * time.Second -const artifactCopyBufferSize = 32 << 10 - -var artifactCopyBufferPool = sync.Pool{ //nolint:gochecknoglobals // shared across artifact responses - New: func() any { - buffer := make([]byte, artifactCopyBufferSize) - return &buffer - }, -} - -// canonicalPackagePURL returns a versionless PURL in canonical form so cooldown -// lookups match keys produced by config.CooldownConfig.NormalizedPackages. -func canonicalPackagePURL(ecosystem, name string) string { - return packageurl.MakeString(ecosystem, name, "") -} - -// canonicalVersionPURL returns a versioned PURL in canonical form, matching -// the keys the artifact cache writes to the versions table. -func canonicalVersionPURL(ecosystem, name, version string) string { - return packageurl.MakeString(ecosystem, name, version) -} - -var errUnsupportedPackageIdentity = errors.New("package identity cannot be represented as a PURL") - -func packagePURLStrings(ecosystem, name, version string) (string, string, error) { - packagePURL := packageurl.MakeString(ecosystem, name, "") - versionPURL := packageurl.MakeString(ecosystem, name, version) - if packagePURL == "" || versionPURL == "" { - return "", "", fmt.Errorf("%w: %s %q", errUnsupportedPackageIdentity, ecosystem, name) - } - return packagePURL, versionPURL, nil -} - const contentTypeJSON = "application/json" -const ( - headerAccept = "Accept" - headerAcceptEncoding = "Accept-Encoding" - headerContentType = "Content-Type" - headerContentLength = "Content-Length" - headerContentEncoding = "Content-Encoding" - headerETag = "ETag" - headerLastModified = "Last-Modified" -) - -// ifNoneMatchHits reports whether the given If-None-Match header value -// matches the current entity tag using weak comparison, so "*" matches any -// tag, W/ prefixes are ignored on both sides, and a comma-separated list is -// scanned. An empty header or an empty stored tag never match. -func ifNoneMatchHits(header, etag string) bool { - if etag == "" || header == "" { - return false - } - if header == "*" { - return true - } - etag = strings.TrimPrefix(etag, "W/") - for value := range strings.SplitSeq(header, ",") { - if strings.TrimPrefix(strings.TrimSpace(value), "W/") == etag { - return true - } - } - return false -} +const headerAcceptEncoding = "Accept-Encoding" // defaultMetadataMaxSize is used when Proxy.MetadataMaxSize is unset. const defaultMetadataMaxSize = 100 << 20 -// defaultMetadataStreamMaxSize is used when Proxy.MetadataStreamMaxSize is -// unset. -const defaultMetadataStreamMaxSize = 1 << 30 - // ErrMetadataTooLarge is returned when upstream metadata exceeds the configured limit. var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit") @@ -153,57 +62,14 @@ var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit") // from unexpectedly large responses. Returns ErrMetadataTooLarge if the response // is truncated by the limit. func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { - return p.ReadMetadataSized(r, -1) -} - -// ReadMetadataSized is ReadMetadata for a body expected to be size bytes -// long, such as a cached document or a response with a Content-Length. -// Reading into a buffer of that size avoids io.ReadAll growing its buffer -// step by step, which peaks at more than twice the document size. size is -// only a hint: a negative one, or one above the limit, reads as ReadMetadata -// does, and a body of a different length is still read correctly. A body -// read through upstreamBodyReader that upstream cuts off fails, even though -// it ends like a body shorter than size. -func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { limit := p.MetadataMaxSize if limit <= 0 { limit = defaultMetadataMaxSize } - if size < 0 || size > limit { - return readLimited(r, nil, limit) - } - // One byte more than expected, so a body of exactly size bytes ends in - // the buffer and a longer one is noticed. - buf := make([]byte, size+1) - n, err := io.ReadFull(r, buf) - var cut *upstreamBodyError - switch { - case errors.As(err, &cut): - return nil, err - case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): - if int64(n) < size/2 { - // The hint was far off; don't keep the oversized buffer alive. - return bytes.Clone(buf[:n]), nil - } - return buf[:n], nil - case err != nil: - return nil, err - } - return readLimited(r, buf, limit) -} - -// readLimited appends the rest of r to data, failing with -// ErrMetadataTooLarge once the total exceeds limit. -func readLimited(r io.Reader, data []byte, limit int64) ([]byte, error) { - rest, err := io.ReadAll(io.LimitReader(r, limit+1-int64(len(data)))) + data, err := io.ReadAll(io.LimitReader(r, limit+1)) if err != nil { return nil, err } - if data == nil { - data = rest - } else { - data = append(data, rest...) - } if int64(len(data)) > limit { return nil, ErrMetadataTooLarge } @@ -212,121 +78,24 @@ func readLimited(r io.Reader, data []byte, limit int64) ([]byte, error) { // Proxy provides shared functionality for protocol handlers. type Proxy struct { - DB *database.DB - Storage storage.Storage - Fetcher fetch.FetcherInterface - Resolver *fetch.Resolver - // RouteFor returns how to fetch a package that doesn't come from the - // ecosystem's default upstream, such as an npm package in - // upstream.npm_routes, or nil to use Resolver. - RouteFor func(ecosystem, name string) *ArtifactRoute - Logger *slog.Logger - Cooldown CooldownPolicy - // ShortestCooldown is the shortest cooldown configured anywhere. It - // bounds how long a rewrite filtered by cooldown is cached. - ShortestCooldown time.Duration - Denylist *denylist.Policy - CacheMetadata bool - MetadataTTL time.Duration - MetadataMaxSize int64 - // MetadataStreamMaxSize caps a metadata document streamed from upstream - // into storage, which metadata_max_size doesn't cover. Default 1 GB. - MetadataStreamMaxSize int64 - GradleReadOnly bool - HostedEnabled bool - GradleMaxUploadSize int64 - // NPMFullMetadata requests full npm packuments from upstream even when - // cooldown is disabled, so served metadata carries publish times. - NPMFullMetadata bool - DirectServe bool - DirectServeTTL time.Duration + DB *database.DB + Storage storage.Storage + Fetcher fetch.FetcherInterface + Resolver *fetch.Resolver + Logger *slog.Logger + Cooldown *cooldown.Config + CacheMetadata bool + MetadataTTL time.Duration + MetadataMaxSize int64 + GradleReadOnly bool + GradleMaxUploadSize int64 + DirectServe bool + DirectServeTTL time.Duration // DirectServeBaseURL, if set, replaces the scheme and host of presigned // URLs so clients receive a public address even when the proxy reaches // storage at an internal one. DirectServeBaseURL string HTTPClient *http.Client - - // RateLimits is the gate under the upstream clients. Artifact fetches - // check it before calling Fetcher, so held hosts never reach its breaker. - RateLimits UpstreamHolds - - // HostedNPMScopes and HostedComposerVendors name namespaces served from - // pushed packages alone, without asking the upstream. - HostedNPMScopes []string - HostedComposerVendors []string - - // StreamArtifacts streams artifacts from upstream without storing them. - // Each request fetches its own copy: there is no cache to check and - // nothing for concurrent misses to share. - StreamArtifacts bool - - // Scanners runs pre-cache artifact scanning (e.g. trivy, ClamAV, Wiz). - // Nil or disabled means artifacts are cached without scanning. - Scanners *scanner.Group - - // ScanSigningKey authenticates pull requests to the internal - // /_internal/scan-fetch route used by scanners to retrieve staged - // artifacts, for every storage backend. - ScanSigningKey []byte - - // ScanFetchBaseURL is the address scanners use to reach this proxy to - // pull staged artifacts. - ScanFetchBaseURL string - - // inFlight coalesces concurrent cache misses for one artifact, so a single - // upstream fetch serves every waiting caller. Keyed by artifactCoalesceKey. - fetchMu sync.Mutex - inFlight map[string]*inflightFetch - - // detachSlots caps the artifact fetches that keep running after their - // client went away (see DetachUpstreamFetch). Guarded by fetchMu while it - // is created. - detachSlots chan struct{} - - // inFlightMeta does the same for metadata misses. Keyed by - // metadataCoalesceKey. - metaMu sync.Mutex - inFlightMeta map[string]*inflightMetadata - - // rewrites caches metadata documents after their handler rewrites them. - // Nil leaves every request to rewrite its own copy. - rewrites *rewriteCache - - // publishIndexes keeps publish times read out of large metadata - // documents for the download cooldown check; see publishIndex. - publishIndexes publishIndexCache - - // now is the clock rewritten metadata is dated by, and started the time - // the process started; see metadataDate. - now func() time.Time - started time.Time - - // cooldownWatch holds the rewrites recording which versions cooldown - // holds back; see cooldown_expiry.go. - cooldownWatch cooldownWatch - shared *Proxy -} - -// cooldownEcosystems are the ecosystems whose handlers apply cooldown. Others -// have no publish time to check, so a configured cooldown has no effect there. -var cooldownEcosystems = map[string]bool{ - "cargo": true, "composer": true, "conda": true, "gem": true, - "golang": true, "helm": true, "hex": true, "maven": true, - "npm": true, "nuget": true, "pub": true, "pypi": true, -} - -// CooldownSupported reports whether the proxy applies cooldown to an -// ecosystem. -func CooldownSupported(ecosystem string) bool { - return cooldownEcosystems[ecosystem] -} - -// CooldownPolicy supplies version-age filtering and package-specific durations. -type CooldownPolicy interface { - IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool - Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision - For(ecosystem, packagePURL string) time.Duration - Enabled() bool } // NewProxy creates a new Proxy with the given dependencies. @@ -343,8 +112,6 @@ func NewProxy(db *database.DB, store storage.Storage, fetcher fetch.FetcherInter HTTPClient: &http.Client{ Timeout: defaultHTTPTimeout, }, - now: time.Now, - started: time.Now(), } } @@ -352,257 +119,93 @@ func NewProxy(db *database.DB, store storage.Storage, fetcher fetch.FetcherInter type CacheResult struct { Reader io.ReadCloser RedirectURL string - Artifact artifacts.Artifact + Size int64 + ContentType string + Hash string Cached bool - storagePath string - // rangeable is set on a HEAD cache hit, which has no Reader, when the - // stored object would open as a seekable reader. - rangeable bool } // GetOrFetchArtifact retrieves an artifact from cache or fetches from upstream. func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { - // NuGet ids are case-insensitive and the NuGet handler stores them - // lowercased, so a mirrored Newtonsoft.Json must land on the same record - // clients are served. - if ecosystem == "nuget" { - name = strings.ToLower(name) - } - pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) - if err != nil { - return nil, err - } - // Deny before resolution as well as before reading the cache. Mirror callers - // do not supply a filename, and resolving it can contact an upstream registry. - if p.Denylist.Denied(versionPURL) { - return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) - } - var info *fetch.ArtifactInfo - if filename == "" { - info, err = p.resolveArtifact(ctx, ecosystem, name, version) - if err != nil { - return nil, err - } - filename = info.Filename - if filename == "" { - return nil, errors.New("resolved artifact has no filename") - } - if route := p.routeFor(ecosystem, name); route != nil { - filename = route.FilenamePrefix + filename - } - } - if p.StreamArtifacts { - if info == nil { - if info, err = p.resolveArtifact(ctx, ecosystem, name, version); err != nil { - return nil, err - } - } - return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, info.URL, "", "", - func(fetchCtx context.Context) (*fetch.Artifact, error) { - return p.Fetcher.Fetch(fetchCtx, info.URL) - }) - } + pkgPURL := purl.MakePURLString(ecosystem, name, "") + versionPURL := purl.MakePURLString(ecosystem, name, version) + if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil { return nil, err } else if cached != nil { return cached, nil } - metrics.RecordArtifactCacheMiss(ecosystem) - key := artifactCoalesceKey(versionPURL, filename, "", "", "") - recheck := func() (artifacts.Artifact, string, bool) { - return p.cachedArtifactRecord(pkgPURL, versionPURL, filename, "") - } - return p.coalesceFetch(ctx, key, recheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { - return p.fetchAndCache(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, info) - }) -} - -// GetCachedArtifact retrieves an artifact from cache without contacting an upstream. -// It returns nil when no usable cache entry exists. -func (p *Proxy) GetCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { - pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) - if err != nil { - return nil, err - } - return p.checkCache(ctx, pkgPURL, versionPURL, filename) -} - -// HasCachedArtifact reports whether an artifact is cached and its object is -// stored, without opening it or counting a cache hit, so a mirror run does -// not look like client traffic. -func (p *Proxy) HasCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) (bool, error) { - pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) - if err != nil { - return false, err - } - if p.Denylist.Denied(versionPURL) { - return false, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) - } - if p.StreamArtifacts { - return false, nil - } - artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) - if err != nil { - return false, fmt.Errorf("checking artifact cache: %w", err) - } - if artifact == nil { - return false, nil - } - // An unusable record is a miss; the fetch that follows clears it. - if _, err := newIntegrityChecks(artifact.Artifact.Digest.Encoded(), artifact.Integrity.String); err != nil { - return false, nil - } - return p.Storage.Exists(ctx, artifact.StoragePath) -} - -// MarkMirrored records that a mirror run found an artifact cached, which -// keeps it from retention without counting as a cache hit. -func (p *Proxy) MarkMirrored(ecosystem, name, version, filename string) error { - _, versionPURL, err := packagePURLStrings(ecosystem, name, version) - if err != nil { - return err - } - return p.DB.MarkArtifactMirrored(versionPURL, filename) -} - -// ClearCachedArtifact clears an artifact cache record after an external -// integrity check fails, and queues its stored bytes for deletion. -func (p *Proxy) ClearCachedArtifact(ecosystem, name, version, filename string) error { - if p.DB == nil || p.Storage == nil { - return nil - } - pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) - if err != nil { - return err - } - cached, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) - if err != nil { - return fmt.Errorf("looking up cached artifact: %w", err) - } - if cached == nil { - return nil - } - return p.DB.DiscardArtifact(versionPURL, filename, cached.StoragePath) + return p.fetchAndCache(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL) } // checkCache looks up an artifact in the cache. Returns nil if not cached. -// With StreamArtifacts set it always reports a miss, so entries stored before the -// mode was enabled are never served. -// -// A miss on a pushed artifact is an error rather than a miss: the caller would -// otherwise fetch the upstream copy and replace the pushed one with it. func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { - result, err := p.lookupCache(ctx, pkgPURL, versionPURL, filename) - if err != nil || result != nil || p.StreamArtifacts { - return result, err - } - return nil, p.localArtifactMiss(versionPURL, filename) -} - -// localArtifactMiss returns ErrHostedArtifactUnavailable when the artifact -// was pushed and is stored but couldn't be served, ErrHostedArtifactMissing -// when it was pushed and its stored object is gone, and nil otherwise. -func (p *Proxy) localArtifactMiss(versionPURL, filename string) error { - record, err := p.DB.GetArtifact(versionPURL, filename) + pkg, err := p.DB.GetPackageByPURL(pkgPURL) if err != nil { - return fmt.Errorf("checking artifact source: %w", err) + return nil, fmt.Errorf("checking package cache: %w", err) } - if record == nil || record.Source != database.SourceLocal { - return nil - } - if record.StoragePath.Valid { - return fmt.Errorf("%w: %s %s", ErrHostedArtifactUnavailable, versionPURL, filename) - } - return fmt.Errorf("%w: %s %s", ErrHostedArtifactMissing, versionPURL, filename) -} - -// lookupCache looks up an artifact in the cache. Returns nil if not cached. -func (p *Proxy) lookupCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { - if p.Denylist.Denied(versionPURL) { - return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) - } - if p.StreamArtifacts { + if pkg == nil { return nil, nil } - if cached, err := p.readStored(ctx, pkgPURL, versionPURL, filename); err != nil || cached != nil { - return cached, err - } - if moved, err := p.rekeyLegacyGoArtifact(pkgPURL, versionPURL, filename); err != nil || !moved { - return nil, err - } - return p.readStored(ctx, pkgPURL, versionPURL, filename) -} -// readStored opens the stored object of an artifact, whether or not -// StreamArtifacts is set. Returns nil if no usable object is stored. -func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { - artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) + ver, err := p.DB.GetVersionByPURL(versionPURL) + if err != nil { + return nil, fmt.Errorf("checking version cache: %w", err) + } + if ver == nil { + return nil, nil + } + + artifact, err := p.DB.GetArtifact(versionPURL, filename) if err != nil { return nil, fmt.Errorf("checking artifact cache: %w", err) } - if artifact == nil { - return nil, nil - } - checks, err := newIntegrityChecks(artifact.Artifact.Digest.Encoded(), artifact.Integrity.String) - if err != nil { - p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err) + if artifact == nil || !artifact.IsCached() { return nil, nil } result := &CacheResult{ - Artifact: artifact.Artifact, + Size: artifact.Size.Int64, + ContentType: artifact.ContentType.String, + Hash: artifact.ContentHash.String, Cached: true, - storagePath: artifact.StoragePath, } if p.DirectServe { - signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL) + signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath.String, p.DirectServeTTL) if err == nil { result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL) - p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) + p.recordCacheHit(pkgPURL, versionPURL, filename) return result, nil } if !errors.Is(err, storage.ErrSignedURLUnsupported) { p.Logger.Warn("failed to sign storage URL, falling back to streaming", - "path", artifact.StoragePath, "error", err) + "path", artifact.StoragePath.String, "error", err) } } - if servingHead(ctx) { - result.rangeable = readsSeekable(p.Storage) - p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) - return result, nil - } - start := time.Now() - reader, err := p.Storage.Open(ctx, artifact.StoragePath) + reader, err := p.Storage.Open(ctx, artifact.StoragePath.String) metrics.RecordStorageOperation("read", time.Since(start)) if err != nil { - metrics.RecordStorageError("read", metrics.FailureDetail{ - PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: err}) + metrics.RecordStorageError("read") p.Logger.Warn("cached artifact missing from storage, will refetch", - "path", artifact.StoragePath, "error", err) + "path", artifact.StoragePath.String, "error", err) return nil, nil } - result.Reader, err = checks.forStoredObject().wrapFailOnMismatch(reader, + result.Reader = newVerifyingReader(reader, artifact.ContentHash.String, ver.Integrity.String, func(reason string) { p.Logger.Error("cached artifact failed integrity check", "purl", versionPURL, "filename", filename, - "path", artifact.StoragePath, "reason", reason) - metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem), metrics.FailureDetail{ - PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: errors.New(reason)}) - if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil { + "path", artifact.StoragePath.String, "reason", reason) + metrics.RecordIntegrityFailure(pkg.Ecosystem) + if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil { p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err) } }) - if err != nil { - _ = reader.Close() - p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err) - return nil, nil - } - p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) + p.recordCacheHit(pkgPURL, versionPURL, filename) return result, nil } @@ -626,68 +229,28 @@ func rewriteSignedURLHost(signed, baseURL string) string { return s.String() } -func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) { +func (p *Proxy) recordCacheHit(pkgPURL, versionPURL, filename string) { _ = p.DB.RecordArtifactHit(versionPURL, filename) - metrics.RecordArtifactCacheHit(ecosystem) -} - -func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, versionPURL, filename string, cause error) { - p.Logger.Warn("cached artifact has unusable integrity metadata", - "purl", versionPURL, "filename", filename, - "path", artifact.StoragePath, "error", cause) - metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem), metrics.FailureDetail{ - PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: cause}) - if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil { - p.Logger.Warn("failed to clear unusable artifact from cache", "error", err) + if parsed, err := purl.Parse(pkgPURL); err == nil { + metrics.RecordCacheHit(purl.PURLTypeToEcosystem(parsed.Type)) } } -// ArtifactRoute fetches a package from an upstream other than its -// ecosystem's default one. -type ArtifactRoute struct { - // Resolver finds the package's download URLs on that upstream. - Resolver *fetch.Resolver - // FilenamePrefix qualifies the artifact filename the way the handler - // does for the route, so clients are served what was fetched. - FilenamePrefix string -} +func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string) (*CacheResult, error) { + // Record cache miss + metrics.RecordCacheMiss(ecosystem) -func (p *Proxy) routeFor(ecosystem, name string) *ArtifactRoute { - if p.RouteFor == nil { - return nil - } - return p.RouteFor(ecosystem, name) -} - -func (p *Proxy) resolveArtifact(ctx context.Context, ecosystem, name, version string) (*fetch.ArtifactInfo, error) { - resolver := p.Resolver - if route := p.routeFor(ecosystem, name); route != nil { - resolver = route.Resolver - } - info, err := resolver.Resolve(ctx, ecosystem, name, version) + // Resolve download URL + info, err := p.Resolver.Resolve(ctx, ecosystem, name, version) if err != nil { - if errors.Is(err, fetch.ErrNotFound) { - return nil, ErrUpstreamNotFound - } return nil, fmt.Errorf("resolving download URL: %w", err) } - return info, nil -} -func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string, info *fetch.ArtifactInfo) (artifacts.Artifact, string, error) { - // Reuse the resolution used for an unnamed cache lookup. Named requests - // still resolve only on a cache miss, inside the coalesced fetch. - if info == nil { - var err error - info, err = p.resolveArtifact(ctx, ecosystem, name, version) - if err != nil { - return artifacts.Artifact{}, "", err - } + // Use resolved filename if provided filename is empty + if filename == "" { + filename = info.Filename } - if err := p.upstreamHeld(info.URL); err != nil { - return artifacts.Artifact{}, "", err - } p.Logger.Info("fetching from upstream", "ecosystem", ecosystem, "name", name, "version", version, "url", info.URL) @@ -698,550 +261,49 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil if err != nil { metrics.RecordUpstreamFetch(ecosystem, fetchDuration) - metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ - PURL: versionPURL, Filename: filename, URL: info.URL, Err: err}) - if errors.Is(err, fetch.ErrNotFound) { - return artifacts.Artifact{}, "", ErrUpstreamNotFound - } - return artifacts.Artifact{}, "", fmt.Errorf("fetching from upstream: %w", err) + metrics.RecordUpstreamError(ecosystem, "fetch_failed") + return nil, fmt.Errorf("fetching from upstream: %w", err) } metrics.RecordUpstreamFetch(ecosystem, fetchDuration) - return p.storeFetched(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, info.URL, "", "", artifact) -} - -// storeArtifact writes a fetched artifact to storage, verifies it against -// upstreamHash and declared, the integrity the registry's metadata declares, -// if they are non-empty, runs it through the scan gate if scanning is enabled, -// and commits it to the cache database, recording declared with the version -// so cache hits are verified against it too. -// -// The scan gate sits between Storage.Store and updateCacheDB: updateCacheDB -// is the only thing that makes an artifact visible to clients (checkCache -// looks up its row before touching Storage), so deferring it until after a -// verdict means a blocked artifact was never reachable by any client. On -// block, the just-stored bytes are deleted and ErrArtifactBlocked is -// returned; updateCacheDB is never called. -// -// It returns the artifact and its storage path, not a reader; callers get one -// from openStoredArtifact. -func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared string, artifact *fetch.Artifact) (artifacts.Artifact, string, error) { - storagePath := storage.FetchPath(ecosystem, name, version, storage.NewFetchID(), filename) - - body, verified, err := verifyIntegrity(artifact.Body, declared) - if err != nil { - _ = artifact.Body.Close() - return artifacts.Artifact{}, "", err - } - + // Store in cache + storagePath := storage.ArtifactPath(ecosystem, "", name, version, filename) storeStart := time.Now() - size, hash, err := p.Storage.Store(ctx, storagePath, body) + size, hash, err := p.Storage.Store(ctx, storagePath, artifact.Body) _ = artifact.Body.Close() metrics.RecordStorageOperation("write", time.Since(storeStart)) + if err != nil { - metrics.RecordStorageError("write", metrics.FailureDetail{ - PURL: versionPURL, Filename: filename, URL: upstreamURL, Path: storagePath, Err: err}) - return artifacts.Artifact{}, "", fmt.Errorf("storing artifact: %w", err) - } - - if !artifactHashMatches(hash, upstreamHash) { - if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { - p.Logger.Warn("failed to discard artifact with mismatched checksum", "path", storagePath, "error", delErr) - } - return artifacts.Artifact{}, "", fmt.Errorf("%w: upstream declared %s, got %s", ErrArtifactDigestMismatch, upstreamHash, hash) - } - if err := verified(); err != nil { - p.Logger.Error("fetched artifact failed integrity check", - "purl", versionPURL, "filename", filename, "url", upstreamURL, "error", err) - metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem), metrics.FailureDetail{ - PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err}) - if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { - p.Logger.Warn("failed to discard artifact with mismatched integrity", "path", storagePath, "error", delErr) - } - return artifacts.Artifact{}, "", err - } - - if p.Scanners != nil && p.Scanners.Enabled() { - if err := p.runScan(ctx, ecosystem, name, version, filename, versionPURL, storagePath, size, artifact.ContentType); err != nil { - // Detached from ctx: a client disconnecting must not abort - // cleanup of a genuinely blocked artifact and leave its bytes - // orphaned in storage with no DB row pointing at them. - if delErr := p.Storage.Delete(context.WithoutCancel(ctx), storagePath); delErr != nil { - p.Logger.Warn("failed to delete blocked artifact from storage", - "path", storagePath, "error", delErr) - } - return artifacts.Artifact{}, "", err - } - } - - sharedArtifact := artifacts.Artifact{ - PURL: versionPURL, - Digest: digest.Digest("sha256:" + hash), - Size: size, - Filename: filename, - MediaType: artifact.ContentType, + metrics.RecordStorageError("write") + return nil, fmt.Errorf("storing artifact: %w", err) } // Update database - if err := p.updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath, sharedArtifact, declared); err != nil { - if errors.Is(err, database.ErrLocalArtifactExists) { - p.Logger.Info("artifact was pushed while it was fetched; keeping the pushed copy", - "purl", versionPURL, "filename", filename) - } else { - p.Logger.Warn("failed to update cache database", "error", err) - } - // Continue anyway - we have the file. Queue it for deletion in case - // no record points at it; the queue skips it while one does. - if qErr := p.DB.QueuePendingDelete(storagePath); qErr != nil && !p.recordedAt(versionPURL, filename, storagePath) { - p.Logger.Warn("failed to queue unrecorded artifact for deletion, deleting it once served", - "path", storagePath, "error", qErr) - return sharedArtifact, storagePath, errUnrecorded - } + if err := p.updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, info.URL, storagePath, hash, size, artifact.ContentType); err != nil { + p.Logger.Warn("failed to update cache database", "error", err) + // Continue anyway - we have the file } - return sharedArtifact, storagePath, nil -} - -// recordedAt reports whether the artifact's record points at storagePath. A -// commit can apply and still report an error, so storeArtifact checks before -// treating an object as unrecorded. A failed lookup counts as not recorded. -func (p *Proxy) recordedAt(versionPURL, filename, storagePath string) bool { - a, err := p.DB.GetArtifact(versionPURL, filename) - return err == nil && a != nil && a.StoragePath.Valid && a.StoragePath.String == storagePath -} - -// errUnrecorded comes back from storeArtifact together with a usable artifact -// and path when neither a record nor a queue entry could be written for it. -// Nothing else would ever delete the object, so the fetch serves it to -// everyone sharing it and deletes it when the last of them is done. -var errUnrecorded = errors.New("stored artifact has no record") - -// unrecordedObject deletes an unrecorded object once every caller sharing -// its fetch has released it. -type unrecordedObject struct { - refs atomic.Int32 - delete func() -} - -func (u *unrecordedObject) release() { - if u.refs.Add(-1) == 0 { - u.delete() - } -} - -// releaseOnClose releases an unrecorded object once its reader is closed. -type releaseOnClose struct { - io.ReadCloser - release func() -} - -func (r *releaseOnClose) Close() error { - err := r.ReadCloser.Close() - r.release() - return err -} - -// seekableReleaseOnClose keeps range requests working on a seekable reader. -type seekableReleaseOnClose struct { - releaseOnClose - io.Seeker -} - -// serveUnrecorded gives one caller its own reader over an unrecorded object -// and releases the caller's hold on it when that reader closes, or right away -// if it can't be opened. -func (p *Proxy) serveUnrecorded(ctx context.Context, u *unrecordedObject, artifact artifacts.Artifact, path string) (*CacheResult, error) { - res, err := p.openStoredArtifact(ctx, artifact, path) - if err != nil { - u.release() - return nil, err - } - // Once, so a second Close can't drop another caller's hold. - r := releaseOnClose{ReadCloser: res.Reader, release: sync.OnceFunc(u.release)} - if s, ok := res.Reader.(io.Seeker); ok { - res.Reader = &seekableReleaseOnClose{releaseOnClose: r, Seeker: s} - } else { - res.Reader = &r - } - return res, nil -} - -// openStoredArtifact gives one caller its own reader over just-committed -// bytes. Callers sharing a fetch cannot share a handle: it has one read -// position, so they would consume each other's bytes and the first Close would -// break the rest. -func (p *Proxy) openStoredArtifact(ctx context.Context, artifact artifacts.Artifact, storagePath string) (*CacheResult, error) { + // Open the stored file to return readStart := time.Now() reader, err := p.Storage.Open(ctx, storagePath) metrics.RecordStorageOperation("read", time.Since(readStart)) if err != nil { - metrics.RecordStorageError("read", metrics.FailureDetail{ - PURL: artifact.PURL, Filename: artifact.Filename, Path: storagePath, Err: err}) + metrics.RecordStorageError("read") return nil, fmt.Errorf("opening cached artifact: %w", err) } return &CacheResult{ - Reader: reader, - Artifact: artifact, - Cached: false, + Reader: reader, + Size: size, + ContentType: artifact.ContentType, + Hash: hash, + Cached: false, }, nil } -// artifactCoalesceKey identifies one artifact fetch. downloadURL, upstreamHash -// and declared are included so callers expecting different bytes (multiple -// upstreams, or a re-published version) never share a fetch. The hash is -// lowercased because artifactHashMatches compares case-insensitively, so one -// digest in two casings describes one artifact and must not split the fetch. -func artifactCoalesceKey(versionPURL, filename, downloadURL, upstreamHash, declared string) string { - return strings.Join([]string{versionPURL, filename, downloadURL, strings.ToLower(upstreamHash), declared}, "\x00") -} - -// cachedArtifactRecord reports an artifact already committed to the cache, -// without opening it. A caller checks the cache before it gets here, so a -// concurrent fetch can commit the same artifact in between; rechecking the -// record keeps that caller from fetching it a second time. A lookup error is -// reported as a miss, which costs a redundant fetch rather than a failure. -func (p *Proxy) cachedArtifactRecord(pkgPURL, versionPURL, filename, upstreamHash string) (artifacts.Artifact, string, bool) { - record, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) - if err != nil || record == nil { - return artifacts.Artifact{}, "", false - } - if !artifactHashMatches(record.Artifact.Digest.Encoded(), upstreamHash) { - return artifacts.Artifact{}, "", false - } - return record.Artifact, record.StoragePath, true -} - -// errSharedFetchAbandoned is what waiters see if the caller running a shared -// fetch panicked out of it. -var errSharedFetchAbandoned = errors.New("shared upstream fetch did not complete") - -// inflightFetch is one upstream fetch that concurrent callers share. val, err -// and unrecorded are written before done closes and read only after, so the -// close is the handoff. done closes under fetchMu, so a waiter leaving early -// knows whether it was counted in waiters. -type inflightFetch struct { - done chan struct{} - val fetchedArtifact - err error - unrecorded *unrecordedObject - - // waiters counts callers waiting on the fetch, guarded by fetchMu. - // Tests read it to know a caller has joined. - waiters int -} - -// coalesceFetch runs commit at most once for concurrent callers sharing key, -// then gives each its own reader over the stored bytes. -// -// The first caller in runs the fetch and the rest wait on it. Roles are -// decided under fetchMu rather than inferred afterwards, because the two need -// different cancellation behaviour: a waiter may leave when its own client goes -// away, while the caller running the fetch must see it through so -// storeArtifact's scan-on-disconnect handling still decides the outcome. -// -// Before fetching, that caller rechecks the cache through recheck: its own -// lookup happened before it took the key, so a fetch that committed in -// between would otherwise be repeated. A hit fills the shared value as a -// fetch would. -// -// commit runs on that caller's context, so cancellation behaves as it did -// uncoalesced and mirroring still relies on it aborting the fetch. If that -// caller goes away, everyone sharing the fetch gets its error and the key is -// released for a later retry. A context marked with DetachUpstreamFetch is the -// exception: there commit keeps running after the caller goes away, so the -// artifact still lands in the cache. -// -// Every sharing caller still records a cache miss, so the gap between -// proxy_cache_misses_total and upstream fetch observations is what coalescing -// saved. -func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { - shared := p.sharedProxy() - shared.fetchMu.Lock() - if shared.inFlight == nil { - shared.inFlight = make(map[string]*inflightFetch) - } - f, joined := shared.inFlight[key] - if joined { - f.waiters++ - } else { - f = &inflightFetch{done: make(chan struct{})} - shared.inFlight[key] = f - } - shared.fetchMu.Unlock() - - if !joined { - if streamsCacheMiss(ctx) { - return p.runSharedFetchStreaming(ctx, key, f, recheck, commit) - } - return p.runSharedFetch(ctx, key, f, recheck, commit) - } - - select { - case <-ctx.Done(): - // This caller gave up; the fetch continues for everyone else. - p.leaveFetch(f) - return nil, ctx.Err() - case <-f.done: - } - if f.err != nil { - return nil, f.err - } - if f.unrecorded != nil { - return p.serveUnrecorded(ctx, f.unrecorded, f.val.artifact, f.val.storagePath) - } - return p.openStoredArtifact(ctx, f.val.artifact, f.val.storagePath) -} - -// leaveFetch withdraws a waiter that gave up. Once the fetch has finished the -// waiter was counted as holding any unrecorded object, so it releases it. -func (p *Proxy) leaveFetch(f *inflightFetch) { - shared := p.sharedProxy() - shared.fetchMu.Lock() - select { - case <-f.done: - shared.fetchMu.Unlock() - if f.unrecorded != nil { - f.unrecorded.release() - } - default: - f.waiters-- - shared.fetchMu.Unlock() - } -} - -// runSharedFetch performs the fetch that joined callers are waiting on. It is -// never abandoned early, and always releases the key and wakes the waiters. -func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { - shared := p.sharedProxy() - // Set before running so a panicking commit leaves waiters with an error - // rather than a zero-valued artifact. - f.err = errSharedFetchAbandoned - defer func() { - shared.fetchMu.Lock() - delete(shared.inFlight, key) - if f.unrecorded != nil { - // Hand each waiter a hold, then drop the one kept while the - // waiters were still unknown. - f.unrecorded.refs.Add(int32(f.waiters)) - defer f.unrecorded.release() - } - close(f.done) - shared.fetchMu.Unlock() - }() - - // A caller checks the cache before reaching here, so a fetch that finished - // in between would otherwise be repeated. Serve that record only if its - // bytes are still present: a record can outlive them, and refetching is - // the same recovery the cache lookup makes. - served := func(ctx context.Context) (*CacheResult, bool) { - stored, path, ok := recheck() - if !ok { - return nil, false - } - res, err := p.openStoredArtifact(ctx, stored, path) - if err != nil { - return nil, false - } - f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, nil - return res, true - } - if res, ok := served(ctx); ok { - return res, nil - } - - // Another instance sharing the database may be fetching the same - // artifact. Wait for it to store the artifact rather than fetch it too. - // The wait makes no upstream request, so a detached fetch waits past - // its client too, bounded by the holder's fetch and the lease expiry; - // otherwise callers that joined this one would get its cancellation. - waitCtx := ctx - if upstreamFetchDetached(ctx) { - waitCtx = context.WithoutCancel(ctx) - } - var waitedFor *CacheResult - release, outcome, err := p.awaitFetchLease(waitCtx, metrics.CacheKindArtifact, key, func() bool { - res, ok := served(waitCtx) - waitedFor = res - return ok - }) - if err != nil { - f.err = err - return nil, err - } - if outcome == leaseStored { - return waitedFor, nil - } - // With leaseFailed, release is a no-op and this instance fetches on its - // own, so a failing upstream costs each instance one attempt in - // parallel. A rate-limited holder has published its hold by now, and - // rereading it makes commit refuse the fetch with the holder's - // Retry-After instead of sending one more request to the limited host. - if outcome == leaseFailed { - p.reloadUpstreamHolds() - } - failed := true - defer func() { release(failed) }() - - commitCtx := ctx - if upstreamFetchDetached(ctx) && p.acquireDetachSlot() { - defer p.releaseDetachSlot() - commitCtx = context.WithoutCancel(ctx) - } - stored, path, err := commit(commitCtx) - unrecorded := errors.Is(err, errUnrecorded) - if unrecorded { - err = nil - } - // Nothing an instance waiting on the lease could find was recorded, so - // an unrecorded object counts as failed for it. - failed = err != nil || unrecorded - f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, err - if err != nil { - return nil, err - } - // A caller handed the stream already has the bytes; opening the stored - // copy would cost a read it never uses, a GET on S3 or GCS. - h := handoffFrom(ctx) - handedOver := h != nil && h.handedOver() - if unrecorded { - f.unrecorded = &unrecordedObject{delete: func() { p.deleteUnrecorded(path) }} - f.unrecorded.refs.Add(1) // until the waiters are counted - if handedOver { - h.holdUnrecorded(f.unrecorded) - return nil, nil - } - f.unrecorded.refs.Add(1) // this caller - return p.serveUnrecorded(ctx, f.unrecorded, stored, path) - } - if handedOver { - return nil, nil - } - return p.openStoredArtifact(ctx, stored, path) -} - -func (p *Proxy) deleteUnrecorded(path string) { - if err := p.Storage.Delete(context.Background(), path); err != nil { - p.Logger.Warn("failed to delete unrecorded artifact", "path", path, "error", err) - } -} - -type servingHeadKey struct{} - -// ServingHead marks ctx as serving a HEAD request. A cache hit under it is -// answered from the artifact's database record without opening the stored -// object: on S3 or GCS opening it issues a GET whose unread body would only -// be thrown away. The server sets it on every HEAD request. -func ServingHead(ctx context.Context) context.Context { - return context.WithValue(ctx, servingHeadKey{}, true) -} - -func servingHead(ctx context.Context) bool { - head, _ := ctx.Value(servingHeadKey{}).(bool) - return head -} - -// readsSeekable reports whether store opens objects as seekable readers, so a -// HEAD response can advertise byte ranges exactly as the GET would. -func readsSeekable(store storage.Storage) bool { - s, ok := store.(storage.SeekableReads) - return ok && s.ReadsSeekable() -} - -type detachUpstreamFetchKey struct{} - -// DetachUpstreamFetch marks ctx so that an artifact fetch started under it -// runs to completion even when ctx is cancelled. The server sets it on client -// requests: a client that gives up on a slow cache miss (pacman aborts after -// 10 s without data) would otherwise cancel the fetch, and every retry would -// start again from zero, so a large artifact never reaches the cache. The -// fetch stays bounded by the artifact HTTP client's timeout, and at most -// maxDetachedFetches run detached at once. Mirror jobs leave it unset so -// cancelling a job still aborts its fetches. -func DetachUpstreamFetch(ctx context.Context) context.Context { - return context.WithValue(ctx, detachUpstreamFetchKey{}, true) -} - -func upstreamFetchDetached(ctx context.Context) bool { - detached, _ := ctx.Value(detachUpstreamFetchKey{}).(bool) - return detached -} - -// maxDetachedFetches caps the artifact fetches that may outlive their client. -// Without a cap a client could start any number of large uncached downloads -// and hang up straight away, leaving each to run to the artifact client's -// timeout. Fetches beyond the cap stay tied to their client, as before -// detaching existed. -const maxDetachedFetches = 64 - -func (p *Proxy) acquireDetachSlot() bool { - shared := p.sharedProxy() - shared.fetchMu.Lock() - if shared.detachSlots == nil { - shared.detachSlots = make(chan struct{}, maxDetachedFetches) - } - slots := shared.detachSlots - shared.fetchMu.Unlock() - - select { - case slots <- struct{}{}: - return true - default: - return false - } -} - -func (p *Proxy) releaseDetachSlot() { - shared := p.sharedProxy() - <-shared.detachSlots -} - -// fetchedArtifact is what a shared fetch hands its callers: metadata and a -// storage path, neither holding reader state. -type fetchedArtifact struct { - artifact artifacts.Artifact - storagePath string -} - -// runScan generates a signed fetch URL for the just-staged artifact and -// asks the configured scanners for a verdict. Returns a wrapped -// ErrArtifactBlocked if any scanner blocks, or a scan-infrastructure error. -// -// The scan call runs on a context detached from ctx's cancellation -// (context.WithoutCancel): ctx is the original client request's context, and -// a client disconnecting mid-scan must not be indistinguishable from a real -// scanner verdict. Group.Scan still bounds the call with its own configured -// timeout, so a detached context cannot hang forever. -func (p *Proxy) runScan(ctx context.Context, ecosystem, name, version, filename, purlStr, storagePath string, size int64, contentType string) error { - fetchURL := p.scanFetchURL(storagePath, p.Scanners.Timeout()) - result := p.Scanners.Scan(context.WithoutCancel(ctx), scanner.Request{ - Ecosystem: ecosystem, - Name: name, - Version: version, - Filename: filename, - PURL: purlStr, - FetchURL: fetchURL, - Size: size, - ContentType: contentType, - }) - if !result.Allowed { - p.Logger.Warn("artifact blocked by security scan", - "ecosystem", ecosystem, "name", name, "version", version, "filename", filename, - "scanner", result.ScannerName, "reason", result.Reason, "infra_error", result.InfraError) - reason := result.Reason - if result.InfraError { - // result.Reason may contain raw scanner-infrastructure details - // (internal hostnames, ports, connection errors) that must not - // reach an untrusted client via the 403 response body. - reason = "scan could not be completed" - } - return fmt.Errorf("%w: %s", ErrArtifactBlocked, reason) - } - return nil -} - -func (p *Proxy) updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath string, artifact artifacts.Artifact, declared string) error { +func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, upstreamURL, storagePath, hash string, size int64, contentType string) error { now := time.Now() // Upsert package @@ -1252,274 +314,103 @@ func (p *Proxy) updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath RegistryURL: sql.NullString{String: upstreamURL, Valid: true}, EnrichedAt: sql.NullTime{Time: now, Valid: true}, } + if err := p.DB.UpsertPackage(pkg); err != nil { + return fmt.Errorf("upserting package: %w", err) + } + // Upsert version ver := &database.Version{ - PURL: artifact.PURL, + PURL: versionPURL, PackagePURL: pkgPURL, - Integrity: sql.NullString{String: declared, Valid: declared != ""}, EnrichedAt: sql.NullTime{Time: now, Valid: true}, } + if err := p.DB.UpsertVersion(ver); err != nil { + return fmt.Errorf("upserting version: %w", err) + } + // Upsert artifact art := &database.Artifact{ - VersionPURL: artifact.PURL, - Filename: artifact.Filename, + VersionPURL: versionPURL, + Filename: filename, UpstreamURL: upstreamURL, StoragePath: sql.NullString{String: storagePath, Valid: true}, - ContentHash: sql.NullString{String: artifact.Digest.Encoded(), Valid: true}, - Size: sql.NullInt64{Int64: artifact.Size, Valid: true}, - ContentType: sql.NullString{String: artifact.MediaType, Valid: true}, + ContentHash: sql.NullString{String: hash, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + ContentType: sql.NullString{String: contentType, Valid: true}, FetchedAt: sql.NullTime{Time: now, Valid: true}, } - // One transaction, so a push that landed while this was fetched keeps - // both its artifact and its version's integrity. - if err := p.DB.RecordCachedArtifact(pkg, ver, art); err != nil { - return fmt.Errorf("recording artifact: %w", err) + if err := p.DB.UpsertArtifact(art); err != nil { + return fmt.Errorf("upserting artifact: %w", err) } + return nil } // ServeArtifact writes a CacheResult to an HTTP response. func ServeArtifact(w http.ResponseWriter, result *CacheResult) { - serveArtifact(w, http.MethodGet, result) -} - -// ServeArtifactRequest writes a CacheResult to an HTTP response using request -// headers such as Range and If-Range. -func ServeArtifactRequest(w http.ResponseWriter, r *http.Request, result *CacheResult) { - if r == nil { - ServeArtifact(w, result) - return - } - serveArtifactResponse(w, r.Method, r, result) -} - -func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { - serveArtifactResponse(w, method, nil, result) -} - -func serveArtifactResponse(w http.ResponseWriter, method string, request *http.Request, result *CacheResult) { - contentHash := "" - if result.Artifact.Digest != "" { - contentHash = result.Artifact.Digest.Encoded() - } if result.RedirectURL != "" { - if contentHash != "" { - w.Header().Set(headerETag, `"`+contentHash+`"`) + if result.Hash != "" { + w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash)) } w.Header().Set("Location", result.RedirectURL) w.WriteHeader(http.StatusFound) return } - if result.Reader != nil { - defer func() { _ = result.Reader.Close() }() - } + defer func() { _ = result.Reader.Close() }() - if result.Artifact.MediaType != "" { - w.Header().Set(headerContentType, result.Artifact.MediaType) + if result.ContentType != "" { + w.Header().Set("Content-Type", result.ContentType) } - if contentHash != "" { - w.Header().Set(headerETag, `"`+contentHash+`"`) + if result.Size > 0 { + w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size)) } - - var seeker io.Seeker - if result.Reader != nil { - seeker, _ = result.Reader.(io.Seeker) - } - if (seeker != nil || result.rangeable) && result.Artifact.Size >= 0 { - w.Header().Set("Accept-Ranges", "bytes") - if seeker != nil && request != nil && serveArtifactRange(w, request, result, seeker) { - return - } - } - - if result.Artifact.Size > 0 || (method == http.MethodHead && result.Artifact.Size == 0) { - w.Header().Set(headerContentLength, strconv.FormatInt(result.Artifact.Size, 10)) + if result.Hash != "" { + w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash)) } w.WriteHeader(http.StatusOK) - if method != http.MethodHead && result.Reader != nil { - copyArtifactBody(w, result.Reader, result.Artifact.Size, false) - } + _, _ = io.Copy(w, result.Reader) } -type parsedByteRange struct { - start int64 - end int64 -} +// ProxyUpstream forwards a request to an upstream URL without caching. +// It copies the request, forwards specified headers, and streams the response back. +// If forwardHeaders is nil, all response headers are copied. +func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamURL string, forwardHeaders []string) { + p.Logger.Debug("proxying to upstream", "url", upstreamURL) -func serveArtifactRange(w http.ResponseWriter, request *http.Request, result *CacheResult, seeker io.Seeker) bool { - if request.Method != http.MethodGet { - return false - } - ranges := request.Header.Values("Range") - if len(ranges) != 1 || !ifRangeMatches(request, w.Header().Get(headerETag)) { - return false - } - - byteRange, valid, satisfiable := parseByteRange(ranges[0], result.Artifact.Size) - if !valid { - return false - } - if !satisfiable { - w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", result.Artifact.Size)) - w.Header().Set(headerContentLength, "0") - w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) - return true - } - - if _, err := seeker.Seek(byteRange.start, io.SeekStart); err != nil { - w.Header().Del(headerContentType) - w.Header().Del(headerContentLength) - w.Header().Del(headerETag) - w.Header().Del("Accept-Ranges") - http.Error(w, "failed to seek cached artifact", http.StatusInternalServerError) - return true - } - - length := byteRange.end - byteRange.start + 1 - w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", byteRange.start, byteRange.end, result.Artifact.Size)) - w.Header().Set(headerContentLength, strconv.FormatInt(length, 10)) - w.WriteHeader(http.StatusPartialContent) - copyArtifactBody(w, result.Reader, length, true) - return true -} - -func ifRangeMatches(request *http.Request, etag string) bool { - values := request.Header.Values("If-Range") - if len(values) == 0 { - return true - } - return len(values) == 1 && etag != "" && strings.TrimSpace(values[0]) == etag -} - -func parseByteRange(value string, size int64) (parsedByteRange, bool, bool) { - unit, spec, ok := strings.Cut(strings.TrimSpace(value), "=") - if !ok || !strings.EqualFold(strings.TrimSpace(unit), "bytes") { - return parsedByteRange{}, false, false - } - spec = strings.TrimSpace(spec) - if spec == "" || strings.Contains(spec, ",") { - return parsedByteRange{}, false, false - } - first, last, ok := strings.Cut(spec, "-") - if !ok { - return parsedByteRange{}, false, false - } - - if first == "" { - suffixLength, ok := parseRangeNumber(last) - if !ok { - return parsedByteRange{}, false, false - } - if size == 0 || suffixLength == 0 { - return parsedByteRange{}, true, false - } - if suffixLength >= size { - return parsedByteRange{start: 0, end: size - 1}, true, true - } - return parsedByteRange{start: size - suffixLength, end: size - 1}, true, true - } - - start, ok := parseRangeNumber(first) - if !ok { - return parsedByteRange{}, false, false - } - if last == "" { - if size == 0 || start >= size { - return parsedByteRange{}, true, false - } - return parsedByteRange{start: start, end: size - 1}, true, true - } - end, ok := parseRangeNumber(last) - if !ok || start > end { - return parsedByteRange{}, false, false - } - if size == 0 || start >= size { - return parsedByteRange{}, true, false - } - if end >= size { - end = size - 1 - } - return parsedByteRange{start: start, end: end}, true, true -} - -func parseRangeNumber(value string) (int64, bool) { - if value == "" { - return 0, false - } - for _, digit := range value { - if digit < '0' || digit > '9' { - return 0, false - } - } - number, err := strconv.ParseInt(value, 10, 64) - return number, err == nil -} - -func copyArtifactBody(w http.ResponseWriter, reader io.Reader, expectedSize int64, bounded bool) { - buffer := artifactCopyBufferPool.Get().(*[]byte) - defer artifactCopyBufferPool.Put(buffer) - - source := reader - if bounded { - source = io.LimitReader(reader, expectedSize) - } - // Hide optional ReaderFrom methods so io.CopyBuffer uses the pooled buffer. - written, err := io.CopyBuffer(struct{ io.Writer }{w}, source, *buffer) - if err != nil || (bounded && written != expectedSize) || (!bounded && expectedSize > 0 && written != expectedSize) { - // Headers are already committed, so an error status is no longer - // possible. Aborting leaves the response unterminated so clients discard - // a truncated or unverified artifact. - // net/http recovers ErrAbortHandler for this request and keeps the server - // running; on HTTP/1.x it may close this connection as well. - panic(http.ErrAbortHandler) - } -} - -// upstreamRequest sends r's method to upstreamURL, copying only the -// forwardHeaders that r sets. -func (p *Proxy) upstreamRequest(r *http.Request, upstreamURL string, forwardHeaders []string) (*http.Response, error) { req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) if err != nil { - return nil, err + http.Error(w, "failed to create request", http.StatusInternalServerError) + return } + + // Copy request headers that affect content negotiation / caching for _, header := range forwardHeaders { if v := r.Header.Get(header); v != "" { req.Header.Set(header, v) } } - return p.HTTPClient.Do(req) -} -// ProxyUpstream forwards a request to an upstream URL without caching. -// It copies the request, forwards specified headers, and streams the response back. -// forwardHeaders controls the request headers sent upstream. End-to-end response -// headers and trailers are relayed independently of that list. -func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamURL string, forwardHeaders []string) { - p.Logger.Debug("proxying to upstream", "url", upstreamURL) - - resp, err := p.upstreamRequest(r, upstreamURL, forwardHeaders) - p.relayUpstream(w, r, resp, err) -} - -// relayUpstream writes the result of an upstream request: the response, or -// a 502 when the request failed. -func (p *Proxy) relayUpstream(w http.ResponseWriter, r *http.Request, resp *http.Response, err error) { + resp, err := p.HTTPClient.Do(req) if err != nil { - if writeUpstreamHeld(w, err) { - return - } p.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return } defer func() { _ = resp.Body.Close() }() - p.relayResponse(w, r, resp, nil) + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) } -// ProxyFile forwards a file request, relaying end-to-end headers and trailers. +// ProxyFile forwards a file request to upstream, copying all response headers. func (p *Proxy) ProxyFile(w http.ResponseWriter, r *http.Request, upstreamURL string) { req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) if err != nil { @@ -1529,481 +420,140 @@ func (p *Proxy) ProxyFile(w http.ResponseWriter, r *http.Request, upstreamURL st resp, err := p.HTTPClient.Do(req) if err != nil { - if writeUpstreamHeld(w, err) { - return - } http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } defer func() { _ = resp.Body.Close() }() - p.relayResponse(w, r, resp, nil) + for key, values := range resp.Header { + for _, v := range values { + w.Header().Add(key, v) + } + } + + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) } // JSONError writes a JSON error response. func JSONError(w http.ResponseWriter, status int, message string) { - w.Header().Set(headerContentType, contentTypeJSON) + w.Header().Set("Content-Type", contentTypeJSON) w.WriteHeader(status) _, _ = fmt.Fprintf(w, `{"error":%q}`, message) } -// staleWarning marks a response served from the metadata cache because the -// upstream could not be reached. -const staleWarning = `110 - "Response is Stale"` - -// markStale sets the stale warning on a metadata response when stale is set. -func markStale(w http.ResponseWriter, stale bool) { - if stale { - w.Header().Set("Warning", staleWarning) - } -} - // ErrUpstreamNotFound indicates the upstream returned 404. -var ErrUpstreamNotFound = fmt.Errorf("upstream: %w", fetch.ErrNotFound) - -// upstreamNotFoundBodyLimit caps the part of an upstream 404 body kept in -// upstreamNotFoundError. -const upstreamNotFoundBodyLimit = 4 << 10 - -// upstreamNotFoundError is ErrUpstreamNotFound with the start of the -// upstream's 404 body, for handlers that relay the reason to clients. -type upstreamNotFoundError struct { - body []byte -} - -func (e *upstreamNotFoundError) Error() string { return ErrUpstreamNotFound.Error() } - -func (e *upstreamNotFoundError) Unwrap() error { return ErrUpstreamNotFound } - -// ErrArtifactBlocked indicates a pre-cache security scan blocked the artifact. -var ErrArtifactBlocked = errors.New("artifact blocked by security scan") - -// ErrVersionDenied indicates an operator explicitly denied this package version. -var ErrVersionDenied = errors.New("package version blocked by denylist") - -// ErrVersionNotFound indicates the requested version is not available through -// the proxy, including when cooldown or other filtering removed it. -var ErrVersionNotFound = errors.New("version not found") - -// ErrPackageNotFound indicates the requested package metadata is not available -// through the proxy or upstream registry. -var ErrPackageNotFound = errors.New("package not found") - -// serveArtifactError writes response for a failed fetch: -// 404 when upstream reports artifact missing, 403 when a security scan or -// denylist blocked the artifact, 502 otherwise. -func (p *Proxy) serveArtifactError(w http.ResponseWriter, err error, clientMsg string) { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if errors.Is(err, ErrArtifactBlocked) || errors.Is(err, ErrVersionDenied) { - JSONError(w, http.StatusForbidden, err.Error()) - return - } - if errors.Is(err, ErrHostedArtifactMissing) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if errors.Is(err, ErrHostedArtifactUnavailable) { - p.Logger.Error("failed to read pushed artifact", "error", err) - http.Error(w, "pushed package is temporarily unavailable", http.StatusServiceUnavailable) - return - } - if writeUpstreamHeld(w, err) { - return - } - p.Logger.Error("failed to get artifact", "error", err) - http.Error(w, clientMsg, http.StatusBadGateway) -} +var ErrUpstreamNotFound = fmt.Errorf("upstream: not found") // errStale304 is returned when upstream sends 304 but the cached file is missing. var errStale304 = fmt.Errorf("upstream returned 304 but cached file is missing") +// metadataStoragePath builds a storage path for cached metadata. +func metadataStoragePath(ecosystem, cacheKey string) string { + return "_metadata/" + ecosystem + "/" + cacheKey + "/metadata" +} + // FetchOrCacheMetadata fetches metadata from upstream with caching. // On success it returns the raw response bytes and content type. // If upstream fails and a cached copy exists, the cached version is returned. // cacheKey is typically the package name but can include subpath components. // Optional acceptHeaders specify the Accept header(s) to send; defaults to application/json. func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL string, acceptHeaders ...string) ([]byte, string, error) { - body, contentType, _, err := p.fetchOrCacheMetadata(ctx, ecosystem, cacheKey, upstreamURL, "", nil, acceptHeaders...) - return body, contentType, err -} + if containsPathTraversal(cacheKey) { + return nil, "", fmt.Errorf("invalid cache key: %q", cacheKey) + } + + storagePath := metadataStoragePath(ecosystem, cacheKey) + + // Check for existing cache entry (for ETag revalidation and TTL) + var entry *database.MetadataCacheEntry + if p.CacheMetadata && p.DB != nil { + entry, _ = p.DB.GetMetadataCache(ecosystem, cacheKey) + } + + // Serve from cache if within TTL (skip upstream entirely) + if entry != nil && p.MetadataTTL > 0 && entry.FetchedAt.Valid { + if time.Since(entry.FetchedAt.Time) < p.MetadataTTL { + cached, readErr := p.Storage.Open(ctx, entry.StoragePath) + if readErr == nil { + defer func() { _ = cached.Close() }() + data, readErr := p.ReadMetadata(cached) + if readErr == nil { + ct := contentTypeJSON + if entry.ContentType.Valid { + ct = entry.ContentType.String + } + return data, ct, nil + } + } + // Cache file missing/unreadable, fall through to upstream + } + } -// fetchOrCacheMetadata implements FetchOrCacheMetadata. acceptEncoding controls -// the upstream Accept-Encoding: an empty string leaves it unset so Go -// transparently decompresses (for direct callers that parse or rewrite the -// body); any non-empty value is sent verbatim, which disables Go's -// decompression so the wire bytes and their Content-Encoding are stored and -// replayed as sent. The ProxyCached path uses "identity" for signed indexes and -// "gzip" where both hops should stay compressed. -// validate, when supplied, runs before caching or serving a document. Validation -// failures follow the same stale-cache fallback path as upstream failures. It -// runs for every caller, including one that shares another caller's fetch, so -// it can also decode the document into request-local state; a caller that -// joined a fetch gets its own validation error directly. It must not modify the -// body, which joined callers share. -func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, acceptEncoding string, validate func([]byte) error, acceptHeaders ...string) ([]byte, string, string, error) { accept := contentTypeJSON if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { accept = acceptHeaders[0] } - res := p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) - return res.body, res.contentType, res.contentEncoding, res.err -} -// fetchMetadataDocument is FetchOrCacheMetadata for callers that rewrite the -// document and need to know which one they got: the result carries its -// digest along with the body. validate, which may be nil, is as for -// fetchOrCacheMetadata. -func (p *Proxy) fetchMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func([]byte) error) metadataResult { - return p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, "", validate) -} - -// lookupMetadata implements fetchOrCacheMetadata and fetchMetadataDocument. -func (p *Proxy) lookupMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error) metadataResult { - if containsPathTraversal(cacheKey) { - return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} - } - - // Serve from cache if within TTL (skip upstream entirely) - if _, hit := p.cachedMetadataState(ctx, ecosystem, cacheKey, validate); hit != nil { - metrics.RecordMetadataCacheHit(ecosystem, len(hit.body)) - return *hit - } - p.recordMetadataCacheMiss(ecosystem) - - return p.coalescedMetadataMiss(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) -} - -// coalescedMetadataMiss handles a metadata cache miss, sharing one upstream -// fetch among concurrent callers with the same key. -func (p *Proxy) coalescedMetadataMiss(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error) metadataResult { - key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate != nil) - res, shared := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { - // The caller's lookup ran before it took the key, so a fetch that - // finished in between has already refreshed the row. Recheck it - // rather than fetching again, and revalidate against the row as it - // is now. - entry, hit := p.cachedMetadataState(fetchCtx, ecosystem, cacheKey, validate) - if hit != nil { - return *hit - } - // Another instance sharing the database may be fetching the same - // document. Only a stored copy within TTL can be served after - // waiting, so without that there is nothing to wait for. - release := func(bool) {} - if p.CacheMetadata && p.MetadataTTL > 0 { - var outcome leaseOutcome - var err error - release, outcome, err = p.awaitFetchLease(fetchCtx, metrics.CacheKindMetadata, key, func() bool { - entry, hit = p.cachedMetadataState(fetchCtx, ecosystem, cacheKey, validate) - return hit != nil - }) - if err != nil { - return metadataResult{err: err} - } - switch outcome { - case leaseStored: - return *hit - case leaseFailed: - // Serve the stale copy at once, as the holder did, rather - // than trying the failing upstream again. Without one, fetch, - // unless the holder was rate limited: its hold, read here, - // refuses the fetch with its Retry-After. - p.reloadUpstreamHolds() - if entry != nil { - if res := p.staleMetadata(fetchCtx, ecosystem, cacheKey, validate, entry, errFetchFailedElsewhere); res.err == nil { - return res - } - } - case leaseFetch: - } - } - res := metadataResult{err: errSharedFetchAbandoned} - defer func() { release(res.err != nil || res.stale) }() - res = p.fetchMetadataFromUpstream(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate, entry) - return res - }) - // The fetch ran the first caller's validate. A caller that joined it runs - // its own on the shared bytes, since validate may also decode them for it. - if shared && res.err == nil && validate != nil { - if err := validate(res.body); err != nil { - return metadataResult{err: err} - } - } - return res -} - -// cachedMetadataState reads the cache row for a metadata lookup. hit is set -// when the row is within TTL and its bytes are usable. Otherwise entry is the -// row to revalidate against with its ETag: nil when there is none, or when -// validate rejected the cached body, since revalidating an unusable body would -// keep it. -func (p *Proxy) cachedMetadataState(ctx context.Context, ecosystem, cacheKey string, validate func([]byte) error) (*database.MetadataCacheEntry, *metadataResult) { - if !p.CacheMetadata || p.DB == nil { - return nil, nil - } - entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) - entry = p.migrateLegacyMetadataEntry(ctx, ecosystem, cacheKey, entry) - if entry != nil && p.MetadataTTL > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL { - data, ct, err := p.readCachedMetadata(ctx, entry, validate) - if err == nil { - hit := metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, fromCache: true}.describedBy(entry) - return entry, &hit - } - if validate != nil { - return nil, nil - } - // Cache file missing/unreadable, fall through to upstream - } - return entry, nil -} - -// refreshMetadata is lookupMetadata for a document whose cached copy is known -// to be out of date although it is within its TTL. It skips the TTL and asks -// upstream, revalidating the cached copy with its validators, and falls back -// to that copy if upstream fails, which the result reports as stale. -// Concurrent refreshes of one document share a fetch. -func (p *Proxy) refreshMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error) metadataResult { - if containsPathTraversal(cacheKey) { - return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} - } - key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate != nil) + "\x00refresh" - res, shared := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { - var entry *database.MetadataCacheEntry - if p.CacheMetadata && p.DB != nil { - entry, _ = p.DB.GetMetadataCache(ecosystem, cacheKey) - } - return p.fetchMetadataFromUpstream(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate, entry) - }) - if shared && res.err == nil && validate != nil { - if err := validate(res.body); err != nil { - return metadataResult{err: err} - } - } - return res -} - -// fetchMetadataFromUpstream fetches metadata after a cache miss, caches it, -// and falls back to the cached copy if upstream fails. entry is the row to -// revalidate against, or nil. -func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error, entry *database.MetadataCacheEntry) metadataResult { - meta, err := p.fetchUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) + // Try upstream + body, contentType, etag, lastModified, err := p.fetchUpstreamMetadata(ctx, upstreamURL, entry, accept) if errors.Is(err, errStale304) { // 304 but cached file is gone; retry without ETag - meta, err = p.fetchUpstreamMetadata(ctx, upstreamURL, nil, accept, acceptEncoding) - } - if err == nil && validate != nil { - err = validate(meta.body) + body, contentType, etag, lastModified, err = p.fetchUpstreamMetadata(ctx, upstreamURL, nil, accept) } if err == nil { - var digest string if p.CacheMetadata { - digest = p.refreshMetadataBlob(ecosystem, cacheKey, meta) - if digest == "" { - digest = p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) - } + p.cacheMetadataBlob(ctx, ecosystem, cacheKey, storagePath, body, contentType, etag, lastModified) } - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} + return body, contentType, nil } - return p.staleMetadata(ctx, ecosystem, cacheKey, validate, entry, err) -} - -// errFetchFailedElsewhere is the upstream error behind a stale copy served -// because another instance's fetch for it failed. -var errFetchFailedElsewhere = errors.New("upstream fetch failed on another instance") - -// staleMetadata falls back to the cached copy in entry after the upstream -// fetch failed with err. -func (p *Proxy) staleMetadata(ctx context.Context, ecosystem, cacheKey string, validate func([]byte) error, entry *database.MetadataCacheEntry, err error) metadataResult { + // Upstream failed -- fall back to cache if available if !p.CacheMetadata || entry == nil { - return metadataResult{err: fmt.Errorf("upstream failed and no cached metadata: %w", err)} + return nil, "", fmt.Errorf("upstream failed and no cached metadata: %w", err) } p.Logger.Warn("upstream metadata fetch failed, checking cache", "ecosystem", ecosystem, "key", cacheKey, "error", err) - // Re-read the row so the encoding describes the blob as it is now: a - // concurrent refetch may have replaced both since entry was read above - // (an identity blob swapped for a gzip one during rollout). - entry = p.currentMetadataEntry(ecosystem, cacheKey, entry) - - data, ct, readErr := p.readCachedMetadata(ctx, entry, validate) + cached, readErr := p.Storage.Open(ctx, entry.StoragePath) if readErr != nil { - return metadataResult{err: fmt.Errorf("upstream failed and cached metadata unusable (%v): %w", readErr, err)} - } - - p.Logger.Info("serving metadata from cache", - "ecosystem", ecosystem, "key", cacheKey) - return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true, upstreamErr: err}.describedBy(entry) -} - -// metadataResult is what a metadata lookup hands back. A shared fetch gives -// every waiter the same body, so callers must treat it as read-only. -type metadataResult struct { - body []byte - contentType string - contentEncoding string - // digest identifies body when the metadata cache stored it, and is - // empty otherwise. - digest string - err error - // stale is set when the body is the cached copy served because upstream - // failed, and upstreamErr is that failure. - stale bool - upstreamErr error - // fromCache is set when the body is a stored copy within its TTL, read - // without asking upstream. - fromCache bool - // entry is the cache row of the stored document, for a lookup that - // leaves the document in storage (see storedMetadataDocument). body is - // nil then. - entry *database.MetadataCacheEntry -} - -// describedBy returns res with the digest that entry, the cache row its body -// was read for, records. -func (res metadataResult) describedBy(entry *database.MetadataCacheEntry) metadataResult { - if entry.ContentDigest.Valid { - res.digest = entry.ContentDigest.String - } - return res -} - -// inflightMetadata is one metadata fetch that concurrent callers share. res is -// written before done closes and read only after, so the close is the handoff. -type inflightMetadata struct { - done chan struct{} - res metadataResult - - // waiters counts callers that joined the fetch, guarded by metaMu. Tests - // read it to know every caller has joined before the fetch finishes. - waiters int -} - -// metadataCoalesceKey identifies metadata requests that can share one fetch. -// Accept and Accept-Encoding are part of it because they change the bytes -// upstream returns: npm serves an abbreviated or a full document for the same -// package, and the cached-proxy paths ask for identity or gzip. Whether the -// caller validates is part of it so an unvalidated caller is never handed a -// result that skipped validation, or the reverse. -func metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validated bool) string { - return strings.Join([]string{ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, strconv.FormatBool(validated)}, "\x00") -} - -// coalesceMetadata runs fetch at most once for concurrent callers sharing key -// and gives each the result, reporting whether this caller joined another's -// fetch rather than running it. It follows coalesceFetch with one difference: -// fetch runs on a context detached from the first caller's cancellation. A -// metadata fetch has no scan or mirror that depends on the caller aborting it, -// and CI jobs asking for the same Composer or npm metadata would otherwise all -// fail when the first of them disconnects. It stays bounded by the HTTP -// client's timeout. Waiters still leave when their own context ends. -func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(context.Context) metadataResult) (metadataResult, bool) { - shared := p.sharedProxy() - shared.metaMu.Lock() - if shared.inFlightMeta == nil { - shared.inFlightMeta = make(map[string]*inflightMetadata) - } - f, joined := shared.inFlightMeta[key] - if joined { - f.waiters++ - } else { - f = &inflightMetadata{done: make(chan struct{})} - shared.inFlightMeta[key] = f - } - shared.metaMu.Unlock() - - if joined { - select { - case <-ctx.Done(): - return metadataResult{err: ctx.Err()}, true - case <-f.done: - return f.res, true - } - } - - // Set before running so a panicking fetch leaves waiters with an error - // rather than an empty body. - f.res = metadataResult{err: errSharedFetchAbandoned} - defer func() { - shared.metaMu.Lock() - delete(shared.inFlightMeta, key) - shared.metaMu.Unlock() - close(f.done) - }() - f.res = fetch(context.WithoutCancel(ctx)) - return f.res, false -} - -func (p *Proxy) readCachedMetadata(ctx context.Context, entry *database.MetadataCacheEntry, validate func([]byte) error) ([]byte, string, error) { - cached, err := p.Storage.Open(ctx, entry.StoragePath) - if err != nil { - return nil, "", err + return nil, "", fmt.Errorf("upstream failed and cached file missing: %w", err) } defer func() { _ = cached.Close() }() - data, err := p.ReadMetadataSized(cached, metadataEntrySize(entry)) - if err == nil && validate != nil { - err = validate(data) + + data, readErr := p.ReadMetadata(cached) + if readErr != nil { + return nil, "", fmt.Errorf("upstream failed and cached read error: %w", err) } - if err != nil { - return nil, "", err - } - p.recordMetadataRead(entry) + ct := contentTypeJSON if entry.ContentType.Valid { ct = entry.ContentType.String } + p.Logger.Info("serving metadata from cache", + "ecosystem", ecosystem, "key", cacheKey) return data, ct, nil } -// metadataEntrySize is the recorded size of a cached metadata document, or -// -1 when none is recorded. -func metadataEntrySize(entry *database.MetadataCacheEntry) int64 { - if !entry.Size.Valid { - return -1 - } - return entry.Size.Int64 -} - -// recordMetadataRead marks a cached metadata document as used, so size -// eviction drops the documents nobody asks for first. -func (p *Proxy) recordMetadataRead(entry *database.MetadataCacheEntry) { - if err := p.DB.RecordMetadataHit(entry.Ecosystem, entry.Name); err != nil { - p.Logger.Warn("failed to record metadata cache hit", "ecosystem", entry.Ecosystem, "key", entry.Name, "error", err) - } -} - -func (p *Proxy) recordMetadataCacheMiss(ecosystem string) { - if p.CacheMetadata { - metrics.RecordMetadataCacheMiss(ecosystem) - } -} - -// upstreamMetadata holds a fetched metadata response in upstream byte form. -type upstreamMetadata struct { - body []byte - contentType string - contentEncoding string - etag string - lastModified time.Time - // notModified is the cache row upstream answered 304 for, whose stored - // bytes body holds. It is nil for a document upstream sent. - notModified *database.MetadataCacheEntry -} - // fetchUpstreamMetadata fetches metadata from upstream, using ETag for conditional revalidation. -// When acceptEncoding is non-empty it is sent as the Accept-Encoding header, which disables Go's -// transparent decompression (it only applies when the transport adds the header itself), so the -// returned bytes are exactly what the upstream sent and any Content-Encoding it applied is reported -// alongside for the caller to store and replay. An empty acceptEncoding leaves Go to negotiate and -// decompress transparently. -func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept, acceptEncoding string) (*upstreamMetadata, error) { - resp, err := p.requestUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) +// Returns the body, content type, ETag, upstream Last-Modified time, and any error. +func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept string) ([]byte, string, string, time.Time, error) { + var zeroTime time.Time + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) if err != nil { - return nil, err + return nil, "", "", zeroTime, fmt.Errorf("creating request: %w", err) + } + req.Header.Set("Accept", accept) + + if entry != nil && entry.ETag.Valid { + req.Header.Set("If-None-Match", entry.ETag.String) + } + + resp, err := p.HTTPClient.Do(req) + if err != nil { + return nil, "", "", zeroTime, fmt.Errorf("fetching metadata: %w", err) } defer func() { _ = resp.Body.Close() }() @@ -2011,176 +561,73 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e if resp.StatusCode == http.StatusNotModified && entry != nil { cached, readErr := p.Storage.Open(ctx, entry.StoragePath) if readErr != nil { - return nil, errStale304 + return nil, "", "", zeroTime, errStale304 } defer func() { _ = cached.Close() }() - data, readErr := p.ReadMetadataSized(cached, metadataEntrySize(entry)) + data, readErr := p.ReadMetadata(cached) if readErr != nil { - return nil, errStale304 + return nil, "", "", zeroTime, errStale304 } - meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, notModified: entry} + ct := contentTypeJSON if entry.ContentType.Valid { - meta.contentType = entry.ContentType.String - } - if entry.ContentEncoding.Valid { - meta.contentEncoding = entry.ContentEncoding.String + ct = entry.ContentType.String } + lm := zeroTime if entry.LastModified.Valid { - meta.lastModified = entry.LastModified.Time + lm = entry.LastModified.Time } - return meta, nil + return data, ct, entry.ETag.String, lm, nil } - if err := upstreamMetadataStatus(resp); err != nil { - return nil, err + if resp.StatusCode == http.StatusNotFound { + return nil, "", "", zeroTime, ErrUpstreamNotFound + } + if resp.StatusCode != http.StatusOK { + return nil, "", "", zeroTime, fmt.Errorf("upstream returned %d", resp.StatusCode) } - // ContentLength is -1 when the transport decompresses the body, and - // otherwise the length of the bytes read here. - body, err := p.ReadMetadataSized(upstreamBodyReader{resp.Body}, resp.ContentLength) + body, err := p.ReadMetadata(resp.Body) if err != nil { - return nil, fmt.Errorf("reading response: %w", err) + return nil, "", "", zeroTime, fmt.Errorf("reading response: %w", err) } - meta := &upstreamMetadata{ - body: body, - contentType: resp.Header.Get(headerContentType), - contentEncoding: resp.Header.Get(headerContentEncoding), - etag: resp.Header.Get(headerETag), + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + contentType = contentTypeJSON } - if meta.contentType == "" { - meta.contentType = contentTypeJSON + + etag := resp.Header.Get("ETag") + + var lastModified time.Time + if lm := resp.Header.Get("Last-Modified"); lm != "" { + lastModified, _ = http.ParseTime(lm) } - if lm := resp.Header.Get(headerLastModified); lm != "" { - meta.lastModified, _ = http.ParseTime(lm) - } - return meta, nil + + return body, contentType, etag, lastModified, nil } -// requestUpstreamMetadata sends the request for a metadata document, -// revalidating entry when it is given. -func (p *Proxy) requestUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept, acceptEncoding string) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) - if err != nil { - return nil, fmt.Errorf("creating request: %w", err) - } - req.Header.Set("Accept", accept) - if acceptEncoding != "" { - req.Header.Set(headerAcceptEncoding, acceptEncoding) - } - - if entry != nil && entry.ETag.Valid { - req.Header.Set("If-None-Match", entry.ETag.String) - } else if entry != nil && entry.LastModified.Valid { - req.Header.Set("If-Modified-Since", entry.LastModified.Time.UTC().Format(http.TimeFormat)) - } - - resp, err := p.HTTPClient.Do(req) - if err != nil { - return nil, fmt.Errorf("fetching metadata: %w", err) - } - return resp, nil -} - -// upstreamMetadataStatus returns the error for a metadata response other -// than 200. Callers handle a 304 they asked for before. -func upstreamMetadataStatus(resp *http.Response) error { - switch resp.StatusCode { - case http.StatusOK: - return nil - case http.StatusNotFound: - body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamNotFoundBodyLimit)) - return &upstreamNotFoundError{body: body} - } - return fmt.Errorf("upstream returned %d", resp.StatusCode) -} - -// refreshMetadataBlob records that upstream answered 304 for the cached -// document meta was read from, by moving the row's fetched_at on. The bytes in -// storage are unchanged, so they are neither written nor hashed again. It -// returns the row's digest, or "" when the caller has to store meta as for a -// new document: when upstream sent one, the row records no digest, or a -// concurrent fetch has replaced the document since it was read. -func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMetadata) string { - entry := meta.notModified - if entry == nil || p.DB == nil || !entry.ContentDigest.Valid { - return "" - } - refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) - if err != nil { - p.Logger.Warn("failed to refresh cached metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return "" - } - if !refreshed { - return "" - } - p.recordMetadataRead(entry) - return entry.ContentDigest.String -} - -// cacheMetadataBlob stores metadata bytes in storage and updates the -// database. It returns the digest it recorded for the bytes, or "" when they -// were not cached. -func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, meta *upstreamMetadata) string { - digest, _ := p.storeMetadataBlob(ctx, ecosystem, cacheKey, storagePath, bytes.NewReader(meta.body), meta) - return digest -} - -// storeMetadataBlob is cacheMetadataBlob for a document read from body, -// which meta describes. It also returns the error that kept it from caching -// the document. -func (p *Proxy) storeMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, body io.Reader, meta *upstreamMetadata) (string, error) { +// cacheMetadataBlob stores metadata bytes in storage and updates the database. +func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, data []byte, contentType, etag string, lastModified time.Time) { if p.DB == nil || p.Storage == nil { - return "", errMetadataNotStored + return } - size, hash, err := p.Storage.Store(ctx, storagePath, body) + size, _, err := p.Storage.Store(ctx, storagePath, bytes.NewReader(data)) if err != nil { p.Logger.Warn("failed to cache metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return "", err - } - var digest string - if hash != "" { - digest = "sha256:" + hash + return } - err = p.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ - Ecosystem: ecosystem, - Name: cacheKey, - StoragePath: storagePath, - ETag: sql.NullString{String: meta.etag, Valid: meta.etag != ""}, - ContentType: sql.NullString{String: meta.contentType, Valid: meta.contentType != ""}, - ContentEncoding: sql.NullString{String: meta.contentEncoding, Valid: meta.contentEncoding != ""}, - // The digest identifies the stored bytes, so a rewrite cached for them - // can be found without reading them back (see storedRewrite). - ContentDigest: sql.NullString{String: digest, Valid: digest != ""}, - Size: sql.NullInt64{Int64: size, Valid: true}, - LastModified: sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + _ = p.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: ecosystem, + Name: cacheKey, + StoragePath: storagePath, + ETag: sql.NullString{String: etag, Valid: etag != ""}, + ContentType: sql.NullString{String: contentType, Valid: contentType != ""}, + Size: sql.NullInt64{Int64: size, Valid: true}, + LastModified: sql.NullTime{Time: lastModified, Valid: !lastModified.IsZero()}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, }) - if err != nil { - // The blob is written but the row describing it is not, so a later - // TTL hit or stale fallback would serve these bytes with the previous - // row's encoding. Drop the blob so row and bytes can never disagree; - // the next request refetches instead. - p.Logger.Warn("failed to record cached metadata, discarding blob", "ecosystem", ecosystem, "key", cacheKey, "error", err) - if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { - p.Logger.Warn("failed to discard metadata blob", "ecosystem", ecosystem, "key", cacheKey, "error", delErr) - } - return "", err - } - return digest, nil -} - -// currentMetadataEntry re-reads the metadata cache row and returns it, or -// fallback when the row cannot be read. Used before serving a stored blob so -// its encoding comes from the row as it is now rather than from a snapshot -// taken before the upstream fetch. -func (p *Proxy) currentMetadataEntry(ecosystem, cacheKey string, fallback *database.MetadataCacheEntry) *database.MetadataCacheEntry { - if fresh, err := p.DB.GetMetadataCache(ecosystem, cacheKey); err == nil && fresh != nil { - return fresh - } - return fallback } // cachedMeta holds cache validators and freshness state from a metadata cache entry. @@ -2199,11 +646,6 @@ func (p *Proxy) lookupCachedMeta(ecosystem, cacheKey string) cachedMeta { if err != nil || entry == nil { return cachedMeta{} } - return p.cachedMetaFor(entry) -} - -// cachedMetaFor returns the cache validators and freshness state of entry. -func (p *Proxy) cachedMetaFor(entry *database.MetadataCacheEntry) cachedMeta { var cm cachedMeta if entry.ETag.Valid { cm.etag = entry.ETag.String @@ -2224,128 +666,65 @@ func (p *Proxy) cachedMetaFor(entry *database.MetadataCacheEntry) cachedMeta { // When metadata caching is disabled, the response is streamed directly to avoid buffering // large metadata responses (e.g. npm packages with many versions) in memory. func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey string, acceptHeaders ...string) { - p.proxyCachedWithEncoding(w, r, upstreamURL, ecosystem, cacheKey, "identity", nil, acceptHeaders...) -} - -// metadataHooks are optional callbacks for proxyCachedWithEncoding: hit runs -// after a fresh cached row is served, with that row and when it was read, and -// fetched runs with a fetched document before it is written. -type metadataHooks struct { - hit func(entry *database.MetadataCacheEntry, seen time.Time) - fetched func(body []byte, contentType, contentEncoding string) -} - -// proxyCachedWithEncoding is ProxyCached with an explicit upstream Accept-Encoding. -// "identity" preserves signed index bytes (the default); "gzip" keeps both hops -// compressed for large, non-hash-pinned metadata whose clients decode gzip -// (Homebrew API and Conda repodata). The stored bytes and Content-Encoding are -// replayed verbatim either way. -func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, acceptEncoding string, hooks *metadataHooks, acceptHeaders ...string) { if !p.CacheMetadata { // Stream directly without buffering when caching is off. - p.proxyMetadataStream(w, r, upstreamURL, acceptEncoding, acceptHeaders...) + p.proxyMetadataStream(w, r, upstreamURL, acceptHeaders...) return } - if hooks != nil && hooks.hit != nil { - if entry := p.freshMetadataEntry(ecosystem, cacheKey); entry != nil { - seen := time.Now() - if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { - hooks.hit(entry, seen) - return - } - } - } else if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { - return - } - - body, contentType, contentEncoding, err := p.fetchOrCacheMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, acceptEncoding, nil, acceptHeaders...) + body, contentType, err := p.FetchOrCacheMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, acceptHeaders...) if err != nil { - p.writeMetadataFetchError(w, err) + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + p.Logger.Error("metadata fetch failed", "error", err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } - if hooks != nil && hooks.fetched != nil { - hooks.fetched(body, contentType, contentEncoding) - } - p.writeMetadataCachedResponseWithEncoding(w, r, ecosystem, cacheKey, body, contentType, contentEncoding) -} - -// writeMetadataFetchError answers a failed metadata fetch. -func (p *Proxy) writeMetadataFetchError(w http.ResponseWriter, err error) { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if writeUpstreamHeld(w, err) { - return - } - p.Logger.Error("metadata fetch failed", "error", err) - http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) + p.writeMetadataCachedResponse(w, r, ecosystem, cacheKey, body, contentType) } // writeMetadataCachedResponse writes a cached metadata response and handles // conditional request headers using metadata cache validators. func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType string) { - p.writeMetadataCachedResponseWithEncoding(w, r, ecosystem, cacheKey, body, contentType, "") -} + cm := p.lookupCachedMeta(ecosystem, cacheKey) -// writeMetadataCachedResponseWithEncoding is writeMetadataCachedResponse with -// an explicit Content-Encoding. contentEncoding must describe the body being -// written; it is passed in rather than re-read from the cache row, which is -// missing or stale when the metadata cache write failed and would otherwise -// mislabel the bytes. -func (p *Proxy) writeMetadataCachedResponseWithEncoding(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType, contentEncoding string) { - writeMetadataBody(w, r, p.lookupCachedMeta(ecosystem, cacheKey), body, contentType, contentEncoding) -} - -// writeMetadataBody writes a metadata document with the validators and -// staleness in cm. -func writeMetadataBody(w http.ResponseWriter, r *http.Request, cm cachedMeta, body []byte, contentType, contentEncoding string) { - if writeMetadataValidators(w, r, cm) { - return - } - - w.Header().Set(headerContentType, contentType) - w.Header().Set(headerContentLength, strconv.Itoa(len(body))) - if contentEncoding != "" { - w.Header().Set(headerContentEncoding, contentEncoding) - } - markStale(w, cm.stale) - w.WriteHeader(http.StatusOK) - if r.Method != http.MethodHead { - _, _ = w.Write(body) - } -} - -// writeMetadataValidators sets cm's ETag and Last-Modified on w and answers -// a conditional request they satisfy with 304, reporting whether it did. -func writeMetadataValidators(w http.ResponseWriter, r *http.Request, cm cachedMeta) bool { if cm.etag != "" { - w.Header().Set(headerETag, cm.etag) - } - if !cm.lastModified.IsZero() { - w.Header().Set(headerLastModified, cm.lastModified.UTC().Format(http.TimeFormat)) - } - if ifNoneMatchHits(r.Header.Get("If-None-Match"), cm.etag) { - w.WriteHeader(http.StatusNotModified) - return true + if match := r.Header.Get("If-None-Match"); match != "" && match == cm.etag { + w.WriteHeader(http.StatusNotModified) + return + } } if !cm.lastModified.IsZero() { if ims := r.Header.Get("If-Modified-Since"); ims != "" { if t, err := http.ParseTime(ims); err == nil && !cm.lastModified.After(t) { w.WriteHeader(http.StatusNotModified) - return true + return } } } - return false + + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + if cm.etag != "" { + w.Header().Set("ETag", cm.etag) + } + if !cm.lastModified.IsZero() { + w.Header().Set("Last-Modified", cm.lastModified.UTC().Format(http.TimeFormat)) + } + if cm.stale { + w.Header().Set("Warning", `110 - "Response is Stale"`) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) } // proxyMetadataStream forwards an upstream metadata response by streaming it to the client // without buffering the full body in memory. -func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upstreamURL, acceptEncoding string, acceptHeaders ...string) { - req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) +func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upstreamURL string, acceptHeaders ...string) { + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) if err != nil { http.Error(w, "failed to create request", http.StatusInternalServerError) return @@ -2356,16 +735,8 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst accept = acceptHeaders[0] } req.Header.Set("Accept", accept) - // Set Accept-Encoding explicitly (identity, or gzip for compressible - // verbatim metadata) so Go does not transparently decompress and strip the - // Content-Encoding of the bytes we forward, regardless of what the client - // negotiated. An empty value leaves the header unset, as in - // fetchUpstreamMetadata. - if acceptEncoding != "" { - req.Header.Set(headerAcceptEncoding, acceptEncoding) - } - for _, header := range []string{"If-Modified-Since", "If-None-Match"} { + for _, header := range []string{headerAcceptEncoding, "If-Modified-Since", "If-None-Match"} { if v := r.Header.Get(header); v != "" { req.Header.Set(header, v) } @@ -2373,468 +744,73 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst resp, err := p.HTTPClient.Do(req) if err != nil { - if writeUpstreamHeld(w, err) { - return - } http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } defer func() { _ = resp.Body.Close() }() - p.relayResponse(w, r, resp, func(dst, src http.Header) { - for _, header := range []string{headerContentType, headerContentLength, headerContentEncoding, headerLastModified, headerETag} { - if v := src.Get(header); v != "" { - dst.Set(header, v) - } + for _, header := range []string{"Content-Type", "Content-Length", "Last-Modified", "ETag"} { + if v := resp.Header.Get(header); v != "" { + w.Header().Set(header, v) } - }) + } + + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) } // GetOrFetchArtifactFromURL retrieves an artifact from cache or fetches from a specific URL. // This is useful for registries where download URLs are determined from metadata. func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string) (*CacheResult, error) { - return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", "") + return p.GetOrFetchArtifactFromURLWithHeaders(ctx, ecosystem, name, version, filename, downloadURL, nil) } // GetOrFetchArtifactFromURLWithHeaders retrieves an artifact from cache or fetches from a URL -// with additional request-specific HTTP headers. +// with additional HTTP headers. This is needed for registries that require authentication +// (e.g. Docker Hub requires a Bearer token even for public images). func (p *Proxy) GetOrFetchArtifactFromURLWithHeaders(ctx context.Context, ecosystem, name, version, filename, downloadURL string, headers http.Header) (*CacheResult, error) { - return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, headers, "", "") -} + pkgPURL := purl.MakePURLString(ecosystem, name, "") + versionPURL := purl.MakePURLString(ecosystem, name, version) -// GetOrFetchArtifactFromURLWithDigest retrieves an artifact and verifies a -// newly fetched response against expected, an OCI-style digest such as -// "sha256:" or "sha512:", before serving or caching it. A digest -// whose algorithm can't be verified is refused with ErrUnsupportedDigest -// rather than fetched unverified. An empty expected fetches without -// verification. -func (p *Proxy) GetOrFetchArtifactFromURLWithDigest(ctx context.Context, ecosystem, name, version, filename, downloadURL, expected string) (*CacheResult, error) { - if expected == "" { - return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", "") - } - upstreamHash, declared, err := digestChecks(expected) - if err != nil { - return nil, err - } - return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, upstreamHash, declared) -} - -// ErrUnsupportedDigest is returned for a digest whose algorithm or encoding -// the proxy can't verify. -var ErrUnsupportedDigest = errors.New("unsupported digest") - -// digestChecks turns an OCI-style digest into the checks the artifact fetch -// runs: the hex of a sha256 digest, which is also how cached artifacts are -// keyed, or for another algorithm the integrity string verifyIntegrity -// checks. -func digestChecks(expected string) (upstreamHash, declared string, _ error) { - // The sha256 hex is compared as given, ignoring case; a malformed one - // fails as a mismatch once the body has been hashed. - if hash, ok := strings.CutPrefix(expected, "sha256:"); ok { - return hash, "", nil - } - d, err := digest.Parse(expected) - if err != nil { - return "", "", fmt.Errorf("%w %q: %w", ErrUnsupportedDigest, expected, err) - } - var algorithm integrity.Algorithm - switch d.Algorithm() { - case digest.SHA384: - algorithm = integrity.SHA384 - case digest.SHA512: - algorithm = integrity.SHA512 - default: - return "", "", fmt.Errorf("%w %q", ErrUnsupportedDigest, expected) - } - sum, err := integrity.ParseHex(algorithm, d.Encoded()) - if err != nil { - return "", "", fmt.Errorf("%w %q: %w", ErrUnsupportedDigest, expected, err) - } - return "", sum.SRI(), nil -} - -// verifyDigest checks body against expected, an OCI-style digest. A digest -// whose algorithm can't be computed is an ErrUnsupportedDigest, and one that -// doesn't match is an ErrArtifactDigestMismatch. -func verifyDigest(expected string, body []byte) error { - d, err := digest.Parse(expected) - if err != nil { - return fmt.Errorf("%w %q: %w", ErrUnsupportedDigest, expected, err) - } - if actual := d.Algorithm().FromBytes(body); actual != d { - return fmt.Errorf("%w: expected %s, got %s", ErrArtifactDigestMismatch, d, actual) - } - return nil -} - -// GetOrFetchArtifactFromURLWithIntegrity retrieves an artifact and verifies a -// newly fetched response against declared, the integrity the registry's -// metadata declares, before serving or caching it: a Subresource Integrity -// string, or a single algorithm-prefixed hex checksum such as "sha1-". -// declared is recorded with the cached version, so cache hits are verified -// against it as well. An empty declared fetches without verification. -func (p *Proxy) GetOrFetchArtifactFromURLWithIntegrity(ctx context.Context, ecosystem, name, version, filename, downloadURL, declared string) (*CacheResult, error) { - return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", declared) -} - -func (p *Proxy) getOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string, headers http.Header, upstreamHash, declared string) (*CacheResult, error) { - pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) - if err != nil { - return nil, err - } - return p.getOrFetchArtifactFromURLWithCachePURLs( - ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers, upstreamHash, declared, - ) -} - -func (p *Proxy) getOrFetchArtifactFromURLWithCachePURLs(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash, declared string) (*CacheResult, error) { - // Some protocols use qualified cache keys. Check their package identity - // too, and the cache key itself for those, such as Swift, whose name has - // no PURL of its own. - if p.versionDenied(ecosystem, name, version) { - return nil, fmt.Errorf("%w: %s", ErrVersionDenied, canonicalVersionPURL(ecosystem, name, version)) - } - if p.Denylist.Denied(versionPURL) { - return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) - } - if p.StreamArtifacts { - return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, downloadURL, upstreamHash, declared, - func(fetchCtx context.Context) (*fetch.Artifact, error) { - return p.fetchUpstream(fetchCtx, downloadURL, headers) - }) - } - if cached, err := p.getCachedArtifactWithUpstreamHash(ctx, pkgPURL, versionPURL, filename, upstreamHash); err != nil { + if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil { return nil, err } else if cached != nil { return cached, nil } - metrics.RecordArtifactCacheMiss(ecosystem) - return p.coalescedFetchFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers, upstreamHash, declared) + + return p.fetchAndCacheFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers) } -// coalescedFetchFromURL fetches an artifact the cache could not serve, sharing -// the fetch with concurrent callers. The caller running it discards a stale -// entry under the key, where it cannot delete a fetch that just replaced it. -func (p *Proxy) coalescedFetchFromURL(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash, declared string) (*CacheResult, error) { - key := artifactCoalesceKey(versionPURL, filename, downloadURL, upstreamHash, declared) - recheck := func() (artifacts.Artifact, string, bool) { - return p.cachedArtifactRecord(pkgPURL, versionPURL, filename, upstreamHash) - } - return p.coalesceFetch(ctx, key, recheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { - p.discardStaleArtifact(pkgPURL, versionPURL, filename, upstreamHash) - return p.fetchAndCacheFromURL(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers, upstreamHash, declared) - }) -} - -// cacheStoredCopy caches a copy of the object stored at source under a new -// name, instead of downloading the same bytes again. It shares the coalescing -// key a download from upstreamURL would use, so the two never run at once. -func (p *Proxy) cacheStoredCopy(ctx context.Context, ecosystem, name, version, filename, upstreamURL, upstreamHash string, source *database.StoredContent) (*CacheResult, error) { - pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) - if err != nil { - return nil, err - } - if p.versionDenied(ecosystem, name, version) || p.Denylist.Denied(versionPURL) { - return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) - } - key := artifactCoalesceKey(versionPURL, filename, upstreamURL, upstreamHash, "") - recheck := func() (artifacts.Artifact, string, bool) { - return p.cachedArtifactRecord(pkgPURL, versionPURL, filename, upstreamHash) - } - return p.coalesceFetch(ctx, key, recheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { - body, err := p.Storage.Open(fetchCtx, source.StoragePath) - if err != nil { - return artifacts.Artifact{}, "", fmt.Errorf("opening stored copy: %w", err) - } - p.Logger.Info("caching a stored copy", "ecosystem", ecosystem, "name", name, "version", version, - "source", source.StoragePath) - return p.storeArtifact(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, "", - &fetch.Artifact{Body: body, Size: source.Size, ContentType: source.ContentType.String}) - }) -} - -// getCachedArtifactWithUpstreamHash returns a cached artifact whose recorded -// content hash matches the checksum the upstream currently declares for it. -// This detects an upstream re-publishing under the same version, which the -// stream integrity check in checkCache cannot: that check only verifies the -// stored blob against the hash recorded when it was cached. A stale entry is -// a miss and is left in place: the fetch that replaces it discards it under -// the coalescing key. -func (p *Proxy) getCachedArtifactWithUpstreamHash(ctx context.Context, pkgPURL, versionPURL, filename, upstreamHash string) (*CacheResult, error) { - cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename) - if err != nil || cached == nil { - return cached, err - } - if artifactHashMatches(cached.Artifact.Digest.Encoded(), upstreamHash) { - return cached, nil - } - if cached.Reader != nil { - _ = cached.Reader.Close() - } - return nil, nil -} - -// discardStaleArtifact clears the cached entry when its digest disagrees with -// upstreamHash, queueing its object for deletion. It runs under the coalescing -// key, after the recheck, so an entry a previous fetch refreshed is kept. -func (p *Proxy) discardStaleArtifact(pkgPURL, versionPURL, filename, upstreamHash string) { - record, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) - if err != nil { - p.Logger.Warn("failed to read cache record before refetch", - "purl", versionPURL, "filename", filename, "error", err) - return - } - if record == nil || artifactHashMatches(record.Artifact.Digest.Encoded(), upstreamHash) { - return - } - p.Logger.Warn("cached artifact hash disagrees with upstream metadata, discarding", - "purl", versionPURL, "filename", filename, "cached", record.Artifact.Digest.Encoded(), "upstream", upstreamHash) - if err := p.DB.DiscardArtifact(versionPURL, filename, record.StoragePath); err != nil { - p.Logger.Warn("failed to clear artifact cache record", "purl", versionPURL, "filename", filename, "error", err) - } -} - -func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash, declared string) (artifacts.Artifact, string, error) { - if err := p.upstreamHeld(downloadURL); err != nil { - return artifacts.Artifact{}, "", err - } +func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header) (*CacheResult, error) { p.Logger.Info("fetching from upstream", "ecosystem", ecosystem, "name", name, "version", version, "url", downloadURL) - fetchStart := time.Now() - artifact, err := p.fetchUpstream(ctx, downloadURL, headers) - metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) + artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) if err != nil { - metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ - PURL: versionPURL, Filename: filename, URL: downloadURL, Err: err}) - if errors.Is(err, fetch.ErrNotFound) { - return artifacts.Artifact{}, "", ErrUpstreamNotFound - } - return artifacts.Artifact{}, "", fmt.Errorf("fetching from upstream: %w", err) - } - - return p.storeFetched(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, upstreamHash, declared, artifact) -} - -type upstreamLastModifiedKey struct{} - -// WithUpstreamLastModified returns a context under which an artifact fetched -// from upstream reports its Last-Modified to fn, so a handler can learn a -// version's publish time from the download itself rather than with a -// request of its own. fn is not called for a cache hit, or when the -// response has no usable Last-Modified. -func WithUpstreamLastModified(ctx context.Context, fn func(time.Time)) context.Context { - return context.WithValue(ctx, upstreamLastModifiedKey{}, fn) -} - -// observedFetcher is the part of fetch.Fetcher and its circuit breaker that -// reports response headers. -type observedFetcher interface { - FetchObservedWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.ObservedArtifact, error) -} - -type signedDownloadKey struct{} - -// signedDownload is a download URL carrying a signature, and the URL without -// its query that stands for it everywhere but the request itself. size, when -// not negative, is the exact size the download must have. -type signedDownload struct { - signed, recorded string - size int64 -} - -// errDownloadSize is a download larger or smaller than the size it was -// expected to have. -var errDownloadSize = errors.New("download does not have the expected size") - -// withSignedDownload returns a context under which a fetch of the returned -// URL requests signedURL instead. The returned URL, signedURL without its -// query and fragment, is what is logged, recorded with the artifact and used -// to coalesce fetches, so a signature that grants access to the object -// stays out of logs and the database. A download of other than size bytes -// is cut off with errDownloadSize, unless size is negative. -func withSignedDownload(ctx context.Context, signedURL string, size int64) (context.Context, string) { - u, err := url.Parse(signedURL) - if err != nil { - return ctx, signedURL - } - u.User = nil - u.RawQuery, u.ForceQuery = "", false - u.Fragment, u.RawFragment = "", "" - recorded := u.String() - return context.WithValue(ctx, signedDownloadKey{}, signedDownload{signed: signedURL, recorded: recorded, size: size}), recorded -} - -// redactedFetchError is a fetch error whose text names the recorded URL -// rather than the signed one. -type redactedFetchError struct { - err error - download signedDownload -} - -func (e *redactedFetchError) Error() string { - return strings.ReplaceAll(e.err.Error(), e.download.signed, e.download.recorded) -} - -func (e *redactedFetchError) Unwrap() error { return e.err } - -// fetchUpstream fetches downloadURL, or the signed URL ctx gives for it, -// reporting its Last-Modified when ctx asks for it. -func (p *Proxy) fetchUpstream(ctx context.Context, downloadURL string, headers http.Header) (*fetch.Artifact, error) { - download, ok := ctx.Value(signedDownloadKey{}).(signedDownload) - if !ok || download.recorded != downloadURL { - return p.fetchUpstreamURL(ctx, downloadURL, headers) - } - artifact, err := p.fetchUpstreamURL(ctx, download.signed, headers) - if err != nil { - return nil, &redactedFetchError{err: err, download: download} - } - if download.size < 0 { - return artifact, nil - } - if artifact.Size >= 0 && artifact.Size != download.size { - _ = artifact.Body.Close() - return nil, fmt.Errorf("%w: upstream announced %d bytes, want %d", errDownloadSize, artifact.Size, download.size) - } - artifact.Body = &exactSizeBody{ReadCloser: artifact.Body, left: download.size} - return artifact, nil -} - -// exactSizeBody fails a body that turns out longer or shorter than left -// bytes, without reading more than one byte past it. -type exactSizeBody struct { - io.ReadCloser - left int64 -} - -func (b *exactSizeBody) Read(p []byte) (int, error) { - if int64(len(p)) > b.left+1 { - p = p[:b.left+1] - } - n, err := b.ReadCloser.Read(p) - b.left -= int64(n) - switch { - case b.left < 0: - return n, fmt.Errorf("%w: upstream sent more bytes", errDownloadSize) - case errors.Is(err, io.EOF) && b.left > 0: - return n, fmt.Errorf("%w: upstream sent %d bytes too few", errDownloadSize, b.left) - } - return n, err -} - -func (p *Proxy) fetchUpstreamURL(ctx context.Context, downloadURL string, headers http.Header) (*fetch.Artifact, error) { - report, _ := ctx.Value(upstreamLastModifiedKey{}).(func(time.Time)) - observer, ok := p.Fetcher.(observedFetcher) - if report == nil || !ok { - return p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) - } - observed, err := observer.FetchObservedWithHeaders(ctx, downloadURL, headers) - if err != nil { - return nil, err - } - if observed.Observation != nil { - if t, err := http.ParseTime(observed.Observation.Headers.Get(headerLastModified)); err == nil { - report(t) - } - } - return observed.Artifact, nil -} - -// streamFromUpstream fetches an artifact when StreamArtifacts is set and hands its -// body to the caller without storing it. -// -// With upstreamHash or the declared integrity set the body is verified as it -// streams. The size is left -// unknown so the response goes out chunked: a mismatch only shows at EOF, -// after the bytes have been sent, and an unterminated chunked response is the -// only way left to make the client reject them (see serveArtifact). -func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version, filename, versionPURL, upstreamURL, upstreamHash, declared string, fetchArtifact func(context.Context) (*fetch.Artifact, error)) (*CacheResult, error) { - if err := p.upstreamHeld(upstreamURL); err != nil { - return nil, err - } - p.Logger.Info("streaming from upstream", - "ecosystem", ecosystem, "name", name, "version", version, "url", upstreamURL) - - fetchStart := time.Now() - artifact, err := fetchArtifact(ctx) - metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) - if err != nil { - metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ - PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err}) - if errors.Is(err, fetch.ErrNotFound) { - return nil, ErrUpstreamNotFound - } return nil, fmt.Errorf("fetching from upstream: %w", err) } - body := &streamErrorLogger{ - ReadCloser: artifact.Body, - onError: func(read int64, err error) { - p.Logger.Warn("streaming artifact from upstream failed", - "purl", versionPURL, "filename", filename, "url", upstreamURL, "bytes", read, "error", err) - metrics.RecordUpstreamError(ecosystem, "stream_failed", metrics.FailureDetail{ - PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err}) - }, - } - result := &CacheResult{ - Reader: body, - Artifact: artifacts.Artifact{ - PURL: versionPURL, - Size: artifact.Size, - Filename: filename, - MediaType: artifact.ContentType, - }, - } - if upstreamHash == "" && declared == "" { - return result, nil - } - - hash := strings.ToLower(upstreamHash) - checks, err := newIntegrityChecks(hash, declared) + storagePath := storage.ArtifactPath(ecosystem, "", name, version, filename) + size, hash, err := p.Storage.Store(ctx, storagePath, artifact.Body) + _ = artifact.Body.Close() if err != nil { - _ = artifact.Body.Close() - return nil, fmt.Errorf("parsing upstream digest: %w", err) + return nil, fmt.Errorf("storing artifact: %w", err) } - result.Reader, err = checks.wrapFailOnMismatch(body, func(reason string) { - p.Logger.Error("streamed artifact failed integrity check", - "purl", versionPURL, "filename", filename, "url", upstreamURL, "reason", reason) - metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem), metrics.FailureDetail{ - PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: errors.New(reason)}) - }) + + if err := p.updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, downloadURL, storagePath, hash, size, artifact.ContentType); err != nil { + p.Logger.Warn("failed to update cache database", "error", err) + } + + reader, err := p.Storage.Open(ctx, storagePath) if err != nil { - _ = artifact.Body.Close() - return nil, err + return nil, fmt.Errorf("opening cached artifact: %w", err) } - if hash != "" { - result.Artifact.Digest = digest.Digest("sha256:" + hash) - } - result.Artifact.Size = -1 - return result, nil -} -// streamErrorLogger reports the first read error of a streamed upstream body. -// The error itself still reaches serveArtifact, which aborts the response. -type streamErrorLogger struct { - io.ReadCloser - onError func(read int64, err error) - read int64 - logged bool -} - -func (r *streamErrorLogger) Read(p []byte) (int, error) { - n, err := r.ReadCloser.Read(p) - r.read += int64(n) - if err != nil && err != io.EOF && !r.logged { - r.logged = true - r.onError(r.read, err) - } - return n, err -} - -// ErrArtifactDigestMismatch indicates that fetched bytes did not match the -// checksum the upstream declared and were not recorded in the cache database. -var ErrArtifactDigestMismatch = errors.New("artifact digest mismatch") - -func artifactHashMatches(got, expected string) bool { - return expected == "" || strings.EqualFold(got, expected) + return &CacheResult{ + Reader: reader, + Size: size, + ContentType: artifact.ContentType, + Hash: hash, + Cached: false, + }, nil } diff --git a/internal/handler/handler_bench_test.go b/internal/handler/handler_bench_test.go deleted file mode 100644 index fdcc935..0000000 --- a/internal/handler/handler_bench_test.go +++ /dev/null @@ -1,326 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "crypto/sha256" - "database/sql" - "encoding/hex" - "fmt" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/storage" - "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries/fetch" -) - -const benchmarkArtifactSize = 64 << 10 - -const benchmarkMetadataSize = 1 << 20 - -type benchmarkResponseWriter struct { - header http.Header -} - -func (w *benchmarkResponseWriter) Header() http.Header { - return w.header -} - -func (w *benchmarkResponseWriter) Write(p []byte) (int, error) { - return len(p), nil -} - -func (w *benchmarkResponseWriter) WriteHeader(_ int) {} - -func benchmarkCachedProxy(b *testing.B) (*Proxy, *mockStorage) { - b.Helper() - - proxy, db, store, _ := setupTestProxy(b) - content := strings.Repeat("x", benchmarkArtifactSize) - seedPackage(b, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", content) - - artifact, err := db.GetArtifact("pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz") - if err != nil { - b.Fatalf("get seeded artifact: %v", err) - } - sum := sha256.Sum256([]byte(content)) - artifact.ContentHash.String = hex.EncodeToString(sum[:]) - if err := db.UpsertArtifact(artifact); err != nil { - b.Fatalf("update seeded artifact hash: %v", err) - } - - return proxy, store -} - -func BenchmarkArtifactCacheHit(b *testing.B) { - ctx := context.Background() - - b.Run("stream-64KiB", func(b *testing.B) { - proxy, _ := benchmarkCachedProxy(b) - w := &benchmarkResponseWriter{header: make(http.Header)} - b.SetBytes(benchmarkArtifactSize) - b.ReportAllocs() - b.ResetTimer() - - for b.Loop() { - result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") - if err != nil { - b.Fatal(err) - } - ServeArtifact(w, result) - } - }) - - b.Run("direct-serve", func(b *testing.B) { - proxy, store := benchmarkCachedProxy(b) - proxy.DirectServe = true - store.signedURL = "https://storage.example/npm/lodash-4.17.21.tgz?signature=abc" - w := &benchmarkResponseWriter{header: make(http.Header)} - b.ReportAllocs() - b.ResetTimer() - - for b.Loop() { - result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") - if err != nil { - b.Fatal(err) - } - ServeArtifact(w, result) - } - }) -} - -func BenchmarkArtifactCacheHitParallel(b *testing.B) { - proxy, _ := benchmarkCachedProxy(b) - ctx := context.Background() - b.SetBytes(benchmarkArtifactSize) - b.ReportAllocs() - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - w := &benchmarkResponseWriter{header: make(http.Header)} - for pb.Next() { - result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") - if err != nil { - b.Error(err) - return - } - ServeArtifact(w, result) - } - }) -} - -func BenchmarkReadMetadata(b *testing.B) { - payload := bytes.Repeat([]byte("x"), benchmarkMetadataSize) - proxy := &Proxy{MetadataMaxSize: benchmarkMetadataSize} - b.SetBytes(benchmarkMetadataSize) - b.ReportAllocs() - - var data []byte - for b.Loop() { - var err error - data, err = proxy.ReadMetadata(bytes.NewReader(payload)) - if err != nil { - b.Fatal(err) - } - } - if len(data) != len(payload) { - b.Fatalf("metadata size = %d, want %d", len(data), len(payload)) - } -} - -// BenchmarkReadMetadataSized reads a large document with and without its -// size known, as for a cached document or a response with Content-Length. -func BenchmarkReadMetadataSized(b *testing.B) { - const size = 64 << 20 - payload := bytes.Repeat([]byte("x"), size) - proxy := &Proxy{MetadataMaxSize: 100 << 20} - for _, bench := range []struct { - name string - hint int64 - }{ - {"unknown size", -1}, - {"known size", size}, - } { - b.Run(bench.name, func(b *testing.B) { - b.SetBytes(size) - b.ReportAllocs() - for b.Loop() { - data, err := proxy.ReadMetadataSized(bytes.NewReader(payload), bench.hint) - if err != nil || len(data) != size { - b.Fatalf("read %d bytes, %v", len(data), err) - } - } - }) - } -} - -func BenchmarkArtifactPURLConstruction(b *testing.B) { - for _, tc := range []struct { - name string - ecosystem string - packageID string - }{ - {"npm", "npm", "lodash"}, - {"scoped-npm", "npm", "@scope/package"}, - {"go", "golang", "github.com/git-pkgs/proxy"}, - } { - b.Run(tc.name, func(b *testing.B) { - b.ReportAllocs() - var packagePURL, versionPURL string - for b.Loop() { - packagePURL = purl.MakePURLString(tc.ecosystem, tc.packageID, "") - versionPURL = purl.MakePURLString(tc.ecosystem, tc.packageID, "1.2.3") - } - if packagePURL == "" || versionPURL == "" { - b.Fatal("empty PURL") - } - }) - } -} - -type benchmarkNPMServer struct { - client *http.Client - requestURL string - db *database.DB - versionPURL string - filename string -} - -func newBenchmarkNPMServer(b *testing.B) *benchmarkNPMServer { - b.Helper() - - ctx := context.Background() - dir := b.TempDir() - db, err := database.Create(filepath.Join(dir, "benchmark.db")) - if err != nil { - b.Fatalf("create database: %v", err) - } - b.Cleanup(func() { _ = db.Close() }) - - store, err := storage.OpenBucket(ctx, "file://"+filepath.Join(dir, "cache")) - if err != nil { - b.Fatalf("open storage: %v", err) - } - b.Cleanup(func() { _ = store.Close() }) - - content := bytes.Repeat([]byte("x"), benchmarkArtifactSize) - storagePath := "npm/lodash/4.17.21/lodash-4.17.21.tgz" - size, hash, err := store.Store(ctx, storagePath, bytes.NewReader(content)) - if err != nil { - b.Fatalf("store artifact: %v", err) - } - - pkg := &database.Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"} - if err := db.UpsertPackage(pkg); err != nil { - b.Fatalf("seed package: %v", err) - } - version := &database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: pkg.PURL} - if err := db.UpsertVersion(version); err != nil { - b.Fatalf("seed version: %v", err) - } - artifact := &database.Artifact{ - VersionPURL: version.PURL, - Filename: "lodash-4.17.21.tgz", - UpstreamURL: "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - StoragePath: sql.NullString{String: storagePath, Valid: true}, - ContentHash: sql.NullString{String: hash, Valid: true}, - Size: sql.NullInt64{Int64: size, Valid: true}, - ContentType: sql.NullString{String: "application/gzip", Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - } - if err := db.UpsertArtifact(artifact); err != nil { - b.Fatalf("seed artifact: %v", err) - } - - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - proxy := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), logger) - handler := NewNPMHandler(proxy, "http://proxy.example", "https://registry.npmjs.org") - server := httptest.NewServer(handler.Routes()) - b.Cleanup(server.Close) - client := server.Client() - return &benchmarkNPMServer{ - client: client, - requestURL: server.URL + "/lodash/-/lodash-4.17.21.tgz", - db: db, - versionPURL: version.PURL, - filename: artifact.Filename, - } -} - -func (s *benchmarkNPMServer) request() error { - resp, err := s.client.Get(s.requestURL) - if err != nil { - return fmt.Errorf("GET cached artifact: %w", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("GET cached artifact status = %d, want %d", resp.StatusCode, http.StatusOK) - } - n, err := io.Copy(io.Discard, resp.Body) - if err != nil { - return fmt.Errorf("read cached artifact: %w", err) - } - if n != benchmarkArtifactSize { - return fmt.Errorf("cached artifact size = %d, want %d", n, benchmarkArtifactSize) - } - return nil -} - -func (s *benchmarkNPMServer) hitCount(b *testing.B) int64 { - b.Helper() - artifact, err := s.db.GetArtifact(s.versionPURL, s.filename) - if err != nil { - b.Fatalf("get artifact hit count: %v", err) - } - return artifact.HitCount -} - -func benchmarkNPMArtifactCacheHitHTTP(b *testing.B, parallel bool) { - server := newBenchmarkNPMServer(b) - if err := server.request(); err != nil { - b.Fatal(err) - } - startHits := server.hitCount(b) - - b.SetBytes(benchmarkArtifactSize) - b.ReportAllocs() - b.ResetTimer() - if parallel { - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - if err := server.request(); err != nil { - b.Error(err) - return - } - } - }) - } else { - for b.Loop() { - if err := server.request(); err != nil { - b.Fatal(err) - } - } - } - b.StopTimer() - - if hitCount := server.hitCount(b) - startHits; hitCount != int64(b.N) { - b.Fatalf("new artifact hits = %d, want %d", hitCount, b.N) - } - b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "requests/s") -} - -func BenchmarkNPMArtifactCacheHitHTTP(b *testing.B) { - benchmarkNPMArtifactCacheHitHTTP(b, false) -} - -func BenchmarkNPMArtifactCacheHitHTTPParallel(b *testing.B) { - benchmarkNPMArtifactCacheHitHTTP(b, true) -} diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 8ce636c..bbcab72 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -3,43 +3,29 @@ package handler import ( "bytes" "context" - "crypto/sha256" "database/sql" "errors" + "fmt" "io" "log/slog" "net/http" "net/http/httptest" - "slices" "strings" - "sync" "testing" "time" - "github.com/git-pkgs/artifacts" - "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/storage" - "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" - "github.com/opencontainers/go-digest" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" - dto "github.com/prometheus/client_model/go" ) // mockStorage implements storage.Storage for testing. type mockStorage struct { - mu sync.Mutex files map[string][]byte storeErr error openErr error signedURL string signErr error - seekable bool - stores int - opens int } func newMockStorage() *mockStorage { @@ -47,8 +33,6 @@ func newMockStorage() *mockStorage { } func (s *mockStorage) Store(_ context.Context, path string, r io.Reader) (int64, string, error) { - s.mu.Lock() - defer s.mu.Unlock() if s.storeErr != nil { return 0, "", s.storeErr } @@ -57,14 +41,10 @@ func (s *mockStorage) Store(_ context.Context, path string, r io.Reader) (int64, return 0, "", err } s.files[path] = data - s.stores++ - return int64(len(data)), sha256Hex(string(data)), nil + return int64(len(data)), "fakehash123", nil } func (s *mockStorage) Open(_ context.Context, path string) (io.ReadCloser, error) { - s.mu.Lock() - defer s.mu.Unlock() - s.opens++ if s.openErr != nil { return nil, s.openErr } @@ -72,44 +52,20 @@ func (s *mockStorage) Open(_ context.Context, path string) (io.ReadCloser, error if !ok { return nil, storage.ErrNotFound } - if s.seekable { - return &mockSeekableReadCloser{Reader: bytes.NewReader(data)}, nil - } return io.NopCloser(bytes.NewReader(data)), nil } -// ReadsSeekable reports what Open returns, as storage.Blob does. -func (s *mockStorage) ReadsSeekable() bool { return s.seekable } - -type mockSeekableReadCloser struct { - *bytes.Reader -} - -func (r *mockSeekableReadCloser) Close() error { return nil } - func (s *mockStorage) Exists(_ context.Context, path string) (bool, error) { - s.mu.Lock() - defer s.mu.Unlock() _, ok := s.files[path] return ok, nil } -func (s *mockStorage) Delete(ctx context.Context, path string) error { - // Real backends (S3/GCS SDKs) fail fast on an already-cancelled - // context; mirror that here so tests can catch cleanup calls that - // forgot to detach from a cancelled client context. - if err := ctx.Err(); err != nil { - return err - } - s.mu.Lock() - defer s.mu.Unlock() +func (s *mockStorage) Delete(_ context.Context, path string) error { delete(s.files, path) return nil } func (s *mockStorage) Size(_ context.Context, path string) (int64, error) { - s.mu.Lock() - defer s.mu.Unlock() data, ok := s.files[path] if !ok { return 0, storage.ErrNotFound @@ -118,8 +74,6 @@ func (s *mockStorage) Size(_ context.Context, path string) (int64, error) { } func (s *mockStorage) UsedSpace(_ context.Context) (int64, error) { - s.mu.Lock() - defer s.mu.Unlock() var total int64 for _, data := range s.files { total += int64(len(data)) @@ -141,30 +95,22 @@ func (s *mockStorage) URL() string { return "mem://" } func (s *mockStorage) Close() error { return nil } -// mockFetcher implements fetch.FetcherInterface for testing. Recording is -// locked because coalescing tests call the handler from many goroutines; tests -// read the recorded fields only after those calls have returned. +// mockFetcher implements fetch.FetcherInterface for testing. type mockFetcher struct { artifact *fetch.Artifact fetchErr error fetchErrByURL map[string]error - - mu sync.Mutex fetchCalled bool fetchedURL string - fetchedHeader http.Header } func (f *mockFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { return f.FetchWithHeaders(ctx, url, nil) } -func (f *mockFetcher) FetchWithHeaders(_ context.Context, url string, headers http.Header) (*fetch.Artifact, error) { - f.mu.Lock() +func (f *mockFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) { f.fetchCalled = true f.fetchedURL = url - f.fetchedHeader = headers.Clone() - f.mu.Unlock() if f.fetchErrByURL != nil { if err, ok := f.fetchErrByURL[url]; ok { return nil, err @@ -181,7 +127,7 @@ func (f *mockFetcher) Head(_ context.Context, _ string) (int64, string, error) { } // setupTestProxy creates a Proxy with a real DB (SQLite in temp dir) and mock storage/fetcher. -func setupTestProxy(t testing.TB) (*Proxy, *database.DB, *mockStorage, *mockFetcher) { +func setupTestProxy(t *testing.T) (*Proxy, *database.DB, *mockStorage, *mockFetcher) { t.Helper() dir := t.TempDir() @@ -200,35 +146,12 @@ func setupTestProxy(t testing.TB) (*Proxy, *database.DB, *mockStorage, *mockFetc return proxy, db, store, fetcher } -func histogramSampleCount(t testing.TB, observer prometheus.Observer) uint64 { - t.Helper() - metric, ok := observer.(prometheus.Metric) - if !ok { - t.Fatal("observer does not implement prometheus.Metric") - } - value := &dto.Metric{} - if err := metric.Write(value); err != nil { - t.Fatalf("writing Prometheus metric: %v", err) - } - return value.GetHistogram().GetSampleCount() -} - -func testArtifact(content, packageURL, filename, mediaType string) artifacts.Artifact { - return artifacts.Artifact{ - PURL: packageURL, - Digest: digest.Digest("sha256:" + sha256Hex(content)), - Size: int64(len(content)), - Filename: filename, - MediaType: mediaType, - } -} - // seedPackage creates a package, version, and cached artifact in the test DB and storage. -func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, name, version, filename, content string) { +func seedPackage(t *testing.T, db *database.DB, store *mockStorage, ecosystem, name, version, filename, content string) { t.Helper() pkg := &database.Package{ - PURL: purl.MakePURLString(ecosystem, name, ""), + PURL: fmt.Sprintf("pkg:%s/%s", ecosystem, name), Ecosystem: ecosystem, Name: name, } @@ -236,7 +159,7 @@ func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, n t.Fatalf("failed to upsert package: %v", err) } - versionPURL := purl.MakePURLString(ecosystem, name, version) + versionPURL := fmt.Sprintf("pkg:%s/%s@%s", ecosystem, name, version) ver := &database.Version{ PURL: versionPURL, PackagePURL: pkg.PURL, @@ -245,16 +168,15 @@ func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, n t.Fatalf("failed to upsert version: %v", err) } - storagePath := ecosystem + "/" + name + "/" + version + "/" + filename + storagePath := storage.ArtifactPath(ecosystem, "", name, version, filename) store.files[storagePath] = []byte(content) - sharedArtifact := testArtifact(content, versionPURL, filename, "application/octet-stream") art := &database.Artifact{ VersionPURL: versionPURL, Filename: filename, UpstreamURL: "https://example.com/" + filename, StoragePath: sql.NullString{String: storagePath, Valid: true}, - ContentHash: sql.NullString{String: sharedArtifact.Digest.Encoded(), Valid: true}, + ContentHash: sql.NullString{String: "abc123", Valid: true}, Size: sql.NullInt64{Int64: int64(len(content)), Valid: true}, ContentType: sql.NullString{String: "application/octet-stream", Valid: true}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, @@ -264,16 +186,6 @@ func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, n } } -// recordedStoragePath returns the storage path the artifact's record points at. -func recordedStoragePath(t testing.TB, db *database.DB, versionPURL, filename string) string { - t.Helper() - art, err := db.GetArtifact(versionPURL, filename) - if err != nil || art == nil || !art.StoragePath.Valid { - t.Fatalf("no storage path recorded for %s %s (err %v)", versionPURL, filename, err) - } - return art.StoragePath.String -} - // pathParseCase holds a single test case for path parsing functions that return // (name, version, arch). type pathParseCase struct { @@ -350,178 +262,16 @@ func TestGetOrFetchArtifact_CacheHit(t *testing.T) { if string(body) != "cached content" { t.Errorf("got body %q, want %q", body, "cached content") } - if result.Artifact.MediaType != "application/octet-stream" { - t.Errorf("got content type %q, want %q", result.Artifact.MediaType, "application/octet-stream") + if result.ContentType != "application/octet-stream" { + t.Errorf("got content type %q, want %q", result.ContentType, "application/octet-stream") } - if result.Artifact.Digest.Encoded() != sha256Hex("cached content") { - t.Errorf("got digest %q, want %q", result.Artifact.Digest.Encoded(), sha256Hex("cached content")) - } -} - -func TestGetCachedArtifactRejectsMalformedIntegrityMetadata(t *testing.T) { - tests := []struct { - name string - malformedHash string - malformedIntegrity string - }{ - {name: "content hash", malformedHash: "abc123"}, - {name: "native integrity", malformedIntegrity: "sha512-abc123"}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assertMalformedCacheRejected(t, test.malformedHash, test.malformedIntegrity) - }) - } -} - -func assertMalformedCacheRejected(t *testing.T, malformedHash, malformedIntegrity string) { - t.Helper() - proxy, db, store, _ := setupTestProxy(t) - const ( - packageName = "broken" - version = "1.0.0" - filename = "broken-1.0.0.tgz" - ) - seedPackage(t, db, store, "npm", packageName, version, filename, "cached content") - versionPURL := purl.MakePURLString("npm", packageName, version) - - if malformedHash != "" { - artifact, err := db.GetArtifact(versionPURL, filename) - if err != nil { - t.Fatal(err) - } - artifact.ContentHash = sql.NullString{String: malformedHash, Valid: true} - if err := db.UpsertArtifact(artifact); err != nil { - t.Fatal(err) - } - } - if malformedIntegrity != "" { - versionRecord := &database.Version{ - PURL: versionPURL, - PackagePURL: purl.MakePURLString("npm", packageName, ""), - Integrity: sql.NullString{String: malformedIntegrity, Valid: true}, - } - if err := db.UpsertVersion(versionRecord); err != nil { - t.Fatal(err) - } - } - - proxy.DirectServe = true - store.signedURL = "https://cache.example/broken" - result, err := proxy.GetCachedArtifact(context.Background(), "npm", packageName, version, filename) - if err != nil { - t.Fatalf("GetCachedArtifact: %v", err) - } - if result != nil { - t.Errorf("GetCachedArtifact = %+v, want nil", result) - } - artifact, err := db.GetArtifact(versionPURL, filename) - if err != nil { - t.Fatal(err) - } - if artifact.StoragePath.Valid { - t.Error("unusable cache record retained its storage path") - } - assertQueuedForDeletion(t, db, "npm/"+packageName+"/"+version+"/"+filename) -} - -// TestCorruptCachedArtifactIsQueuedForDeletion streams cached bytes that no -// longer match their recorded digest, which clears the record once the stream -// ends. With each fetch writing its own path, no later fetch overwrites them, -// so they must be queued for deletion. -func TestCorruptCachedArtifactIsQueuedForDeletion(t *testing.T) { - proxy, db, store, _ := setupTestProxy(t) - seedPackage(t, db, store, "npm", "corrupt", "1.0.0", "corrupt-1.0.0.tgz", "cached content") - storagePath := "npm/corrupt/1.0.0/corrupt-1.0.0.tgz" - store.files[storagePath] = []byte("tampered bytes") - - result, err := proxy.GetCachedArtifact(context.Background(), "npm", "corrupt", "1.0.0", "corrupt-1.0.0.tgz") - if err != nil || result == nil { - t.Fatalf("GetCachedArtifact = %v, %v", result, err) - } - drain(result) - artifact, err := db.GetArtifact("pkg:npm/corrupt@1.0.0", "corrupt-1.0.0.tgz") - if err != nil || artifact == nil || artifact.StoragePath.Valid { - t.Errorf("record = %+v (err %v), want it cleared", artifact, err) - } - assertQueuedForDeletion(t, db, storagePath) -} - -// TestCachedArtifactVerifiedByContentHash checks that a cache hit is verified -// against the recorded content hash only. The version's native integrity is -// one value per version and was checked when the bytes were stored; a stale -// or foreign value there must not discard an artifact whose bytes still -// match their SHA-256. -func TestCachedArtifactVerifiedByContentHash(t *testing.T) { - proxy, db, store, _ := setupTestProxy(t) - seedPackage(t, db, store, "npm", "hashed", "1.0.0", "hashed-1.0.0.tgz", "cached content") - setVersionIntegrity(t, db, "npm", "hashed", "1.0.0", sha512SRI("other bytes")) - - result, err := proxy.GetCachedArtifact(context.Background(), "npm", "hashed", "1.0.0", "hashed-1.0.0.tgz") - if err != nil || result == nil { - t.Fatalf("GetCachedArtifact = %v, %v", result, err) - } - body, err := io.ReadAll(result.Reader) - _ = result.Reader.Close() - if err != nil || string(body) != "cached content" { - t.Fatalf("read %q, %v; want the cached bytes", body, err) - } - artifact, err := db.GetArtifact("pkg:npm/hashed@1.0.0", "hashed-1.0.0.tgz") - if err != nil || artifact == nil || !artifact.StoragePath.Valid { - t.Errorf("record = %+v (err %v), want it kept", artifact, err) - } -} - -// TestCachedArtifactWithoutContentHashChecksNativeIntegrity checks that a -// record without a content hash is still verified against the native -// integrity, which is then the only digest there is. -func TestCachedArtifactWithoutContentHashChecksNativeIntegrity(t *testing.T) { - proxy, db, store, _ := setupTestProxy(t) - seedPackage(t, db, store, "npm", "nohash", "1.0.0", "nohash-1.0.0.tgz", "cached content") - setVersionIntegrity(t, db, "npm", "nohash", "1.0.0", sha512SRI("other bytes")) - artifact, err := db.GetArtifact("pkg:npm/nohash@1.0.0", "nohash-1.0.0.tgz") - if err != nil { - t.Fatal(err) - } - artifact.ContentHash = sql.NullString{} - if err := db.UpsertArtifact(artifact); err != nil { - t.Fatal(err) - } - - result, err := proxy.GetCachedArtifact(context.Background(), "npm", "nohash", "1.0.0", "nohash-1.0.0.tgz") - if err != nil || result == nil { - t.Fatalf("GetCachedArtifact = %v, %v", result, err) - } - _, err = io.ReadAll(result.Reader) - _ = result.Reader.Close() - if !errors.Is(err, ErrArtifactDigestMismatch) { - t.Fatalf("read error = %v, want ErrArtifactDigestMismatch", err) - } -} - -func setVersionIntegrity(t *testing.T, db *database.DB, ecosystem, name, version, integrity string) { - t.Helper() - if err := db.UpsertVersion(&database.Version{ - PURL: purl.MakePURLString(ecosystem, name, version), - PackagePURL: purl.MakePURLString(ecosystem, name, ""), - Integrity: sql.NullString{String: integrity, Valid: true}, - }); err != nil { - t.Fatal(err) - } -} - -// assertQueuedForDeletion fails unless path waits in the pending delete queue. -func assertQueuedForDeletion(t *testing.T, db *database.DB, path string) { - t.Helper() - queued, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 100) - if err != nil || !slices.Contains(queued, path) { - t.Errorf("queued %v (err %v), want %q queued for deletion", queued, err, path) + if result.Hash != "abc123" { + t.Errorf("got hash %q, want %q", result.Hash, "abc123") } } func TestGetOrFetchArtifact_CacheMiss_NoPackage(t *testing.T) { proxy, _, _, fetcher := setupTestProxy(t) - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("npm", metrics.CacheKindArtifact)) // The resolver will fail because "nonexistent" isn't a real package, // but we're testing that it tries to fetch (doesn't return from cache). @@ -531,10 +281,6 @@ func TestGetOrFetchArtifact_CacheMiss_NoPackage(t *testing.T) { if err == nil { t.Fatal("expected error for uncached package") } - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("npm", metrics.CacheKindArtifact)) - if diff := missesAfter - missesBefore; diff != 1 { - t.Errorf("cache misses delta = %.0f, want 1", diff) - } } func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) { @@ -550,7 +296,7 @@ func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) { Filename: "missing-1.0.0.tgz", UpstreamURL: "https://example.com/missing.tgz", StoragePath: sql.NullString{String: "nonexistent/path.tgz", Valid: true}, - ContentHash: sql.NullString{String: sha256Hex("missing content"), Valid: true}, + ContentHash: sql.NullString{String: "hash", Valid: true}, Size: sql.NullInt64{Int64: 100, Valid: true}, ContentType: sql.NullString{String: "application/octet-stream", Valid: true}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, @@ -577,34 +323,12 @@ func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) { } // Verify the new content was stored - storagePath := recordedStoragePath(t, db, "pkg:npm/missing@1.0.0", "missing-1.0.0.tgz") + storagePath := storage.ArtifactPath("npm", "", "missing", "1.0.0", "missing-1.0.0.tgz") if _, ok := store.files[storagePath]; !ok { t.Error("refetched artifact should be stored") } } -func TestArtifactCacheRejectsUnsupportedPackageIdentity(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - - _, err := proxy.GetCachedArtifact( - context.Background(), "swift", "apple/example", "1.2.3", "example-1.2.3.zip", - ) - if !errors.Is(err, errUnsupportedPackageIdentity) { - t.Fatalf("GetCachedArtifact() error = %v, want unsupported package identity", err) - } - - _, err = proxy.GetOrFetchArtifactFromURL( - context.Background(), "swift", "apple/example", "1.2.3", "example-1.2.3.zip", - "https://registry.example/apple/example/1.2.3.zip", - ) - if !errors.Is(err, errUnsupportedPackageIdentity) { - t.Fatalf("GetOrFetchArtifactFromURL() error = %v, want unsupported package identity", err) - } - if fetcher.fetchCalled { - t.Error("unsupported package identity reached the artifact fetcher") - } -} - func TestGetOrFetchArtifact_DirectServe_Redirect(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") @@ -774,10 +498,8 @@ func TestServeArtifact_Redirect(t *testing.T) { w := httptest.NewRecorder() ServeArtifact(w, &CacheResult{ RedirectURL: "https://bucket.s3.amazonaws.com/file?sig=abc", - Artifact: artifacts.Artifact{ - Digest: digest.Digest("sha256:" + strings.Repeat("a", sha256.Size*2)), - }, - Cached: true, + Hash: "abc123", + Cached: true, }) if w.Code != http.StatusFound { @@ -786,8 +508,8 @@ func TestServeArtifact_Redirect(t *testing.T) { if loc := w.Header().Get("Location"); loc != "https://bucket.s3.amazonaws.com/file?sig=abc" { t.Errorf("Location = %q", loc) } - if etag := w.Header().Get("ETag"); etag != `"`+strings.Repeat("a", sha256.Size*2)+`"` { - t.Errorf("ETag = %q", etag) + if etag := w.Header().Get("ETag"); etag != `"abc123"` { + t.Errorf("ETag = %q, want %q", etag, `"abc123"`) } if cl := w.Header().Get("Content-Length"); cl != "" { t.Errorf("Content-Length should not be set on redirect, got %q", cl) @@ -797,13 +519,10 @@ func TestServeArtifact_Redirect(t *testing.T) { func TestServeArtifact_Stream(t *testing.T) { w := httptest.NewRecorder() ServeArtifact(w, &CacheResult{ - Reader: io.NopCloser(strings.NewReader("payload")), - Artifact: testArtifact( - "payload", - "pkg:npm/example@1.0.0", - "example.tgz", - "application/octet-stream", - ), + Reader: io.NopCloser(strings.NewReader("payload")), + Size: 7, + ContentType: "application/octet-stream", + Hash: "abc123", }) if w.Code != http.StatusOK { @@ -817,165 +536,9 @@ func TestServeArtifact_Stream(t *testing.T) { } } -func TestServeArtifactRequestRanges(t *testing.T) { - const payload = "hello world" - tests := []struct { - name string - rangeHeader string - secondRange string - wantStatus int - wantRange string - wantLength string - wantBody string - }{ - {name: "bounded", rangeHeader: "bytes=1-3", wantStatus: http.StatusPartialContent, wantRange: "bytes 1-3/11", wantLength: "3", wantBody: "ell"}, - {name: "open ended", rangeHeader: "bytes=6-", wantStatus: http.StatusPartialContent, wantRange: "bytes 6-10/11", wantLength: "5", wantBody: "world"}, - {name: "suffix", rangeHeader: "bytes=-4", wantStatus: http.StatusPartialContent, wantRange: "bytes 7-10/11", wantLength: "4", wantBody: "orld"}, - {name: "suffix larger than artifact", rangeHeader: "bytes=-99", wantStatus: http.StatusPartialContent, wantRange: "bytes 0-10/11", wantLength: "11", wantBody: payload}, - {name: "unsatisfiable", rangeHeader: "bytes=11-", wantStatus: http.StatusRequestedRangeNotSatisfiable, wantRange: "bytes */11", wantLength: "0"}, - {name: "malformed", rangeHeader: "bytes=invalid", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, - {name: "reversed", rangeHeader: "bytes=4-2", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, - {name: "multiple ranges", rangeHeader: "bytes=0-1,4-5", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, - {name: "multiple range fields", rangeHeader: "bytes=0-1", secondRange: "bytes=4-5", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, - {name: "no range", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - request := httptest.NewRequest(http.MethodGet, "/artifact", nil) - if test.rangeHeader != "" { - request.Header.Set("Range", test.rangeHeader) - } - if test.secondRange != "" { - request.Header.Add("Range", test.secondRange) - } - w := httptest.NewRecorder() - ServeArtifactRequest(w, request, newRangeTestResult(payload)) - - if w.Code != test.wantStatus { - t.Errorf("status = %d, want %d", w.Code, test.wantStatus) - } - if got := w.Header().Get("Content-Range"); got != test.wantRange { - t.Errorf("Content-Range = %q, want %q", got, test.wantRange) - } - if got := w.Header().Get(headerContentLength); got != test.wantLength { - t.Errorf("Content-Length = %q, want %q", got, test.wantLength) - } - if got := w.Body.String(); got != test.wantBody { - t.Errorf("body = %q, want %q", got, test.wantBody) - } - if got := w.Header().Get("Accept-Ranges"); got != "bytes" { - t.Errorf("Accept-Ranges = %q, want bytes", got) - } - }) - } -} - -func TestServeArtifactRequestIfRange(t *testing.T) { - const payload = "hello world" - tests := []struct { - name string - ifRange string - wantStatus int - wantRange string - wantBody string - }{ - {name: "matching etag", ifRange: `"sha256-matching"`, wantStatus: http.StatusPartialContent, wantRange: "bytes 0-4/11", wantBody: "hello"}, - {name: "mismatching etag", ifRange: `"sha256-stale"`, wantStatus: http.StatusOK, wantBody: payload}, - {name: "weak etag", ifRange: `W/"sha256-matching"`, wantStatus: http.StatusOK, wantBody: payload}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result := newRangeTestResult(payload) - request := httptest.NewRequest(http.MethodGet, "/artifact", nil) - request.Header.Set("Range", "bytes=0-4") - if test.name == "matching etag" { - test.ifRange = `"` + result.Artifact.Digest.Encoded() + `"` - } - request.Header.Set("If-Range", test.ifRange) - w := httptest.NewRecorder() - ServeArtifactRequest(w, request, result) - - if w.Code != test.wantStatus { - t.Errorf("status = %d, want %d", w.Code, test.wantStatus) - } - if got := w.Header().Get("Content-Range"); got != test.wantRange { - t.Errorf("Content-Range = %q, want %q", got, test.wantRange) - } - if got := w.Body.String(); got != test.wantBody { - t.Errorf("body = %q, want %q", got, test.wantBody) - } - }) - } -} - -func TestServeArtifactRequestHeadIgnoresRange(t *testing.T) { - request := httptest.NewRequest(http.MethodHead, "/artifact", nil) - request.Header.Set("Range", "bytes=0-1") - w := httptest.NewRecorder() - ServeArtifactRequest(w, request, newRangeTestResult("hello world")) - - if w.Code != http.StatusOK { - t.Errorf("status = %d, want %d", w.Code, http.StatusOK) - } - if got := w.Header().Get(headerContentLength); got != "11" { - t.Errorf("Content-Length = %q, want 11", got) - } - if got := w.Header().Get("Content-Range"); got != "" { - t.Errorf("Content-Range = %q, want empty", got) - } - if w.Body.Len() != 0 { - t.Errorf("HEAD body = %q, want empty", w.Body.String()) - } -} - -func TestServeArtifactRequestWithoutSeekCapability(t *testing.T) { - request := httptest.NewRequest(http.MethodGet, "/artifact", nil) - request.Header.Set("Range", "bytes=1-2") - w := httptest.NewRecorder() - ServeArtifactRequest(w, request, &CacheResult{ - Reader: io.NopCloser(strings.NewReader("payload")), - Artifact: testArtifact("payload", "pkg:npm/example@1.0.0", "example.tgz", "application/gzip"), - }) - - if w.Code != http.StatusOK || w.Body.String() != "payload" { - t.Errorf("response = %d %q, want 200 with full payload", w.Code, w.Body.String()) - } - if got := w.Header().Get("Accept-Ranges"); got != "" { - t.Errorf("Accept-Ranges = %q, want empty", got) - } -} - -func TestServeArtifactRequestRedirectDoesNotAdvertiseRanges(t *testing.T) { - request := httptest.NewRequest(http.MethodGet, "/artifact", nil) - request.Header.Set("Range", "bytes=1-2") - w := httptest.NewRecorder() - ServeArtifactRequest(w, request, &CacheResult{RedirectURL: "https://storage.example/artifact"}) - - if w.Code != http.StatusFound { - t.Errorf("status = %d, want %d", w.Code, http.StatusFound) - } - if got := w.Header().Get("Accept-Ranges"); got != "" { - t.Errorf("Accept-Ranges = %q, want empty", got) - } -} - -type rangeTestReadSeeker struct { - *bytes.Reader -} - -func (r *rangeTestReadSeeker) Close() error { return nil } - -func newRangeTestResult(payload string) *CacheResult { - return &CacheResult{ - Reader: &rangeTestReadSeeker{Reader: bytes.NewReader([]byte(payload))}, - Artifact: testArtifact(payload, "pkg:npm/example@1.0.0", "example.tgz", "application/gzip"), - } -} - func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) seedPackage(t, db, store, "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "pypi content") - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi", metrics.CacheKindArtifact)) result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "https://pypi.org/files/requests-2.28.0.tar.gz") if err != nil { @@ -989,18 +552,10 @@ func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) { if fetcher.fetchCalled { t.Error("fetcher should not be called on cache hit") } - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi", metrics.CacheKindArtifact)) - if diff := missesAfter - missesBefore; diff != 0 { - t.Errorf("cache misses delta = %.0f, want 0", diff) - } } func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi", metrics.CacheKindArtifact)) - fetchesBefore := histogramSampleCount(t, metrics.UpstreamFetchDuration.WithLabelValues("pypi")) - writesBefore := histogramSampleCount(t, metrics.StorageOperationDuration.WithLabelValues("write")) - readsBefore := histogramSampleCount(t, metrics.StorageOperationDuration.WithLabelValues("read")) + proxy, _, store, fetcher := setupTestProxy(t) fetcher.artifact = &fetch.Artifact{ Body: io.NopCloser(strings.NewReader("fetched content")), @@ -1027,43 +582,17 @@ func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) { if string(body) != "fetched content" { t.Errorf("got body %q, want %q", body, "fetched content") } - if err := result.Artifact.Validate(); err != nil { - t.Errorf("Artifact.Validate() error = %v", err) - } - if result.Artifact.PURL != "pkg:pypi/newpkg@1.0.0" { - t.Errorf("PURL = %q", result.Artifact.PURL) - } - if result.Artifact.Size != int64(len("fetched content")) { - t.Errorf("Size = %d", result.Artifact.Size) - } - if result.Artifact.MediaType != "application/gzip" { - t.Errorf("MediaType = %q", result.Artifact.MediaType) - } // Verify it was stored - storagePath := recordedStoragePath(t, db, "pkg:pypi/newpkg@1.0.0", "newpkg-1.0.0.tar.gz") + storagePath := storage.ArtifactPath("pypi", "", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz") if _, ok := store.files[storagePath]; !ok { t.Error("artifact was not stored in storage") } - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi", metrics.CacheKindArtifact)) - if diff := missesAfter - missesBefore; diff != 1 { - t.Errorf("cache misses delta = %.0f, want 1", diff) - } - if diff := histogramSampleCount(t, metrics.UpstreamFetchDuration.WithLabelValues("pypi")) - fetchesBefore; diff != 1 { - t.Errorf("upstream fetch observations delta = %d, want 1", diff) - } - if diff := histogramSampleCount(t, metrics.StorageOperationDuration.WithLabelValues("write")) - writesBefore; diff != 1 { - t.Errorf("storage write observations delta = %d, want 1", diff) - } - if diff := histogramSampleCount(t, metrics.StorageOperationDuration.WithLabelValues("read")) - readsBefore; diff != 1 { - t.Errorf("storage read observations delta = %d, want 1", diff) - } } func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) { proxy, _, _, fetcher := setupTestProxy(t) fetcher.fetchErr = errors.New("connection refused") - errorsBefore := testutil.ToFloat64(metrics.UpstreamErrors.WithLabelValues("pypi", "fetch_failed")) _, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "fail", "1.0.0", "fail-1.0.0.tar.gz", "https://pypi.org/files/fail-1.0.0.tar.gz") if err == nil { @@ -1072,20 +601,11 @@ func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) { if !strings.Contains(err.Error(), "fetching from upstream") { t.Errorf("expected upstream error, got: %v", err) } - if diff := testutil.ToFloat64(metrics.UpstreamErrors.WithLabelValues("pypi", "fetch_failed")) - errorsBefore; diff != 1 { - t.Errorf("upstream errors delta = %.0f, want 1", diff) - } - recent := metrics.RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "pypi", "error_type": "fetch_failed"}) - if len(recent) == 0 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || - recent[0].URL != "https://pypi.org/files/fail-1.0.0.tar.gz" || recent[0].Error != "connection refused" { - t.Errorf("upstream failures = %+v, want the newest failure to contain the failed package, its URL and the error", recent) - } } func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) { proxy, _, store, fetcher := setupTestProxy(t) store.storeErr = errors.New("disk full") - errorsBefore := testutil.ToFloat64(metrics.StorageErrors.WithLabelValues("write")) fetcher.artifact = &fetch.Artifact{ Body: io.NopCloser(strings.NewReader("data")), ContentType: "application/gzip", @@ -1098,21 +618,15 @@ func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) { if !strings.Contains(err.Error(), "storing artifact") { t.Errorf("expected storage error, got: %v", err) } - if diff := testutil.ToFloat64(metrics.StorageErrors.WithLabelValues("write")) - errorsBefore; diff != 1 { - t.Errorf("storage errors delta = %.0f, want 1", diff) - } - recent := metrics.RecentFailures("proxy_storage_errors_total", map[string]string{"operation": "write"}) - if len(recent) == 0 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || recent[0].Filename != "fail-1.0.0.tar.gz" || - recent[0].Path == "" || recent[0].Error != "disk full" { - t.Errorf("storage failures = %+v, want the newest failure to contain the failed package, its storage path and the error", recent) - } } func TestServeArtifact(t *testing.T) { result := &CacheResult{ - Reader: io.NopCloser(strings.NewReader("file contents")), - Artifact: testArtifact("file contents", "pkg:npm/example@1.0.0", "example.tgz", "application/gzip"), - Cached: true, + Reader: io.NopCloser(strings.NewReader("file contents")), + Size: 13, + ContentType: "application/gzip", + Hash: "sha256abc", + Cached: true, } w := httptest.NewRecorder() @@ -1127,9 +641,8 @@ func TestServeArtifact(t *testing.T) { if w.Header().Get("Content-Length") != "13" { t.Errorf("Content-Length = %q, want %q", w.Header().Get("Content-Length"), "13") } - wantETag := `"` + result.Artifact.Digest.Encoded() + `"` - if w.Header().Get("ETag") != wantETag { - t.Errorf("ETag = %q, want %q", w.Header().Get("ETag"), wantETag) + if w.Header().Get("ETag") != `"sha256abc"` { + t.Errorf("ETag = %q, want %q", w.Header().Get("ETag"), `"sha256abc"`) } if w.Body.String() != "file contents" { t.Errorf("body = %q, want %q", w.Body.String(), "file contents") @@ -1364,10 +877,6 @@ func TestProxyCached_NoValidators_OmitsHeaders(t *testing.T) { } func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) { - hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test", metrics.CacheKindMetadata)) - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test", metrics.CacheKindMetadata)) - artifactHitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test", metrics.CacheKindArtifact)) - servedBefore := testutil.ToFloat64(metrics.MetadataServedBytes.WithLabelValues("test")) upstreamHits := 0 upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { upstreamHits++ @@ -1394,12 +903,6 @@ func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) { if upstreamHits != 1 { t.Fatalf("expected 1 upstream hit, got %d", upstreamHits) } - if diff := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test", metrics.CacheKindMetadata)) - missesBefore; diff != 1 { - t.Errorf("cache misses delta after first request = %.0f, want 1", diff) - } - if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test", metrics.CacheKindMetadata)) - hitsBefore; diff != 0 { - t.Errorf("cache hits delta after first request = %.0f, want 0", diff) - } // Second request within TTL should serve from cache without hitting upstream body, _, err = proxy.FetchOrCacheMetadata(ctx, "test", "ttl-pkg", upstream.URL+"/pkg") @@ -1412,22 +915,9 @@ func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) { if upstreamHits != 1 { t.Errorf("expected upstream to still be hit only once, got %d", upstreamHits) } - if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test", metrics.CacheKindMetadata)) - hitsBefore; diff != 1 { - t.Errorf("cache hits delta after second request = %.0f, want 1", diff) - } - if diff := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test", metrics.CacheKindMetadata)) - missesBefore; diff != 1 { - t.Errorf("cache misses delta after second request = %.0f, want 1", diff) - } - if diff := testutil.ToFloat64(metrics.MetadataServedBytes.WithLabelValues("test")) - servedBefore; diff != float64(len(body)) { - t.Errorf("metadata served bytes delta = %.0f, want %d (only the cached response)", diff, len(body)) - } - if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test", metrics.CacheKindArtifact)) - artifactHitsBefore; diff != 0 { - t.Errorf("artifact cache hits delta = %.0f, want 0 for a metadata lookup", diff) - } } func TestFetchOrCacheMetadata_TTL_Zero_AlwaysRevalidates(t *testing.T) { - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test", metrics.CacheKindMetadata)) upstreamHits := 0 upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { upstreamHits++ @@ -1456,40 +946,6 @@ func TestFetchOrCacheMetadata_TTL_Zero_AlwaysRevalidates(t *testing.T) { if upstreamHits != 2 { t.Errorf("expected 2 upstream hits with TTL=0, got %d", upstreamHits) } - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test", metrics.CacheKindMetadata)) - if diff := missesAfter - missesBefore; diff != 2 { - t.Errorf("cache misses delta = %.0f, want 2", diff) - } -} - -func TestFetchOrCacheMetadata_CacheDisabledDoesNotRecordMetrics(t *testing.T) { - const ecosystem = "metadata-disabled" - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"v":1}`)) - })) - t.Cleanup(upstream.Close) - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - - hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues(ecosystem, metrics.CacheKindMetadata)) - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues(ecosystem, metrics.CacheKindMetadata)) - - _, _, err := proxy.FetchOrCacheMetadata(context.Background(), ecosystem, "pkg", upstream.URL+"/pkg") - if err != nil { - t.Fatalf("fetch metadata: %v", err) - } - - hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues(ecosystem, metrics.CacheKindMetadata)) - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues(ecosystem, metrics.CacheKindMetadata)) - if diff := hitsAfter - hitsBefore; diff != 0 { - t.Errorf("cache hits delta = %.0f, want 0", diff) - } - if diff := missesAfter - missesBefore; diff != 0 { - t.Errorf("cache misses delta = %.0f, want 0", diff) - } } func TestProxyCached_StaleWarningHeader(t *testing.T) { @@ -1554,33 +1010,3 @@ func TestProxyCached_FreshResponse_NoWarningHeader(t *testing.T) { t.Errorf("Warning should be empty for fresh response, got %q", got) } } - -// TestCanonicalPackagePURLMatchesConfig ensures the runtime cooldown lookup key -// agrees with config.CooldownConfig.NormalizedPackages for the same package, -// so a configured override is actually found regardless of how the user wrote it. -func TestCanonicalPackagePURLMatchesConfig(t *testing.T) { - tests := []struct { - ecosystem string - requestName string - configKey string - }{ - {"npm", "@babel/core", "pkg:npm/@babel/core"}, - {"npm", "@babel/core", "pkg:npm/%40babel/core"}, - {"npm", "@typescript/typescript-darwin-arm64", "pkg:npm/@typescript/typescript-darwin-arm64"}, - {"pypi", "Django", "pkg:pypi/Django"}, - {"pypi", "django", "pkg:pypi/Django"}, - {"composer", "symfony/console", "pkg:composer/Symfony/Console"}, - {"cargo", "serde", "pkg:cargo/serde"}, - } - for _, tt := range tests { - t.Run(tt.ecosystem+"/"+tt.requestName+"<="+tt.configKey, func(t *testing.T) { - cfg := config.CooldownConfig{Packages: map[string]string{tt.configKey: "1d"}} - normalized := cfg.NormalizedPackages() - - lookup := canonicalPackagePURL(tt.ecosystem, tt.requestName) - if _, ok := normalized[lookup]; !ok { - t.Errorf("lookup key %q not found in normalized config %v", lookup, normalized) - } - }) - } -} diff --git a/internal/handler/head_cache_hit_test.go b/internal/handler/head_cache_hit_test.go deleted file mode 100644 index 3bcdf6c..0000000 --- a/internal/handler/head_cache_hit_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "maps" - "net/http" - "net/http/httptest" - "testing" - - "github.com/git-pkgs/purl" -) - -// A HEAD on a cache hit is answered from the artifact's database record. It -// must not open the stored object (a GET on S3 or GCS), and its headers must -// be the ones the GET sends. -func TestHeadCacheHitSkipsStorageOpen(t *testing.T) { - const digest = "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" - for _, seekable := range []bool{false, true} { - t.Run(fmt.Sprintf("seekable=%t", seekable), func(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - store.seekable = seekable - seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") - h := &ContainerHandler{ - proxy: proxy, - registryURL: "https://registry.invalid", - proxyURL: "http://localhost:8080", - } - target := "/library/nginx/blobs/" + digest - - get := httptest.NewRecorder() - h.Routes().ServeHTTP(get, httptest.NewRequest(http.MethodGet, target, nil)) - if get.Code != http.StatusOK || get.Body.String() != "cached blob" { - t.Fatalf("GET: status %d, body %q", get.Code, get.Body.String()) - } - opensAfterGet := store.opens - - req := httptest.NewRequest(http.MethodHead, target, nil) - req = req.WithContext(ServingHead(req.Context())) - head := httptest.NewRecorder() - h.Routes().ServeHTTP(head, req) - - if head.Code != http.StatusOK { - t.Fatalf("HEAD: status %d, body %q", head.Code, head.Body.String()) - } - if got := store.opens - opensAfterGet; got != 0 { - t.Errorf("HEAD opened the stored object %d times, want 0", got) - } - if head.Body.Len() != 0 { - t.Errorf("HEAD body length = %d, want 0", head.Body.Len()) - } - if !maps.EqualFunc(get.Header(), head.Header(), func(a, b []string) bool { - return fmt.Sprint(a) == fmt.Sprint(b) - }) { - t.Errorf("HEAD headers differ from GET:\n GET %v\n HEAD %v", get.Header(), head.Header()) - } - if got := get.Header().Get("Accept-Ranges") == "bytes"; got != seekable { - t.Errorf("GET Accept-Ranges set = %t, want %t", got, seekable) - } - if fetcher.fetchCalled { - t.Error("fetcher called on a cache hit") - } - - // A HEAD still counts as a hit, as it did when it opened the object. - art, err := db.GetArtifact(purl.MakePURLString("oci", "library/nginx", digest), digest) - if err != nil || art == nil { - t.Fatalf("artifact record: %v", err) - } - if art.HitCount != 2 { - t.Errorf("hit count = %d, want 2 (GET and HEAD)", art.HitCount) - } - }) - } -} - -// A request that is not marked as HEAD reads the object as before. -func TestCacheHitWithoutHeadMarkOpensObject(t *testing.T) { - proxy, db, store, _ := setupTestProxy(t) - seedPackage(t, db, store, "npm", "demo", "1.0.0", "demo.tgz", "content") - result, err := proxy.GetCachedArtifact(context.Background(), "npm", "demo", "1.0.0", "demo.tgz") - if err != nil || result == nil || result.Reader == nil { - t.Fatalf("result=%+v err=%v; want an open reader", result, err) - } - _ = result.Reader.Close() - if store.opens != 1 { - t.Errorf("opens = %d, want 1", store.opens) - } -} diff --git a/internal/handler/helm.go b/internal/handler/helm.go deleted file mode 100644 index 661ca98..0000000 --- a/internal/handler/helm.go +++ /dev/null @@ -1,574 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "net/http" - "net/url" - "path" - "strings" - "sync" - "time" - - "gopkg.in/yaml.v3" - - "github.com/git-pkgs/proxy/internal/retention" -) - -const ( - helmMetadataEcosystem = "helm" - helmIndexFilename = "index.yaml" - sha256HexLength = 64 -) - -// Helm takes no package rules: charts are cached under the configured -// repository's name and the chart digest, not under the chart name, so a key -// like pkg:helm/nginx would name a repository. The helm rule and the default -// cover charts from HTTP repositories; charts pulled over OCI go through the -// container handler and fall under oci. -func init() { - retention.Default.Register(retention.Spec{Key: "helm"}) -} - -// HelmHandler serves read-only HTTP Helm chart repositories. Each configured -// repository is mounted at /helm/{repository}/. -type HelmHandler struct { - proxy *Proxy - proxyURL string - repositories map[string]string - ociRegistries []helmOCIRegistry - - lookupMu sync.Mutex - // lookups holds the charts of each repository's last parsed index. - lookups map[string]*helmChartLookup -} - -// NewHelmHandler creates a Helm chart repository protocol handler. -func NewHelmHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *HelmHandler { - h := &HelmHandler{ - proxyURL: strings.TrimSuffix(proxyURL, "/"), - repositories: make(map[string]string, len(repositories)), - proxy: proxy, - } - for name, repositoryURL := range repositories { - h.repositories[name] = strings.TrimSuffix(repositoryURL, "/") - } - return h -} - -// Routes returns the HTTP handler for Helm chart repository requests. -func (h *HelmHandler) Routes() http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("GET /{repository}/index.yaml", h.handleIndex) - mux.HandleFunc("GET /{repository}/charts/{digest}/{filename}", h.handleChart) - return mux -} - -func (h *HelmHandler) handleIndex(w http.ResponseWriter, r *http.Request) { - repository, upstreamURL, ok := h.repositoryForRequest(r) - if !ok { - http.NotFound(w, r) - return - } - - // Parsing and encoding a large index costs far more than serving it, so - // rewrites go through the rewrite cache like npm packuments. The cache - // key holds the repository and upstream URL, both part of cacheKey. - cacheKey := h.indexCacheKey(repository, upstreamURL) - if stored, contentType, ok := h.proxy.storedRewriteVariant(helmMetadataEcosystem, cacheKey, h.proxyURL, cacheKey, nil); ok { - serveRewrittenMetadataAs(w, r, stored, contentType) - return - } - - index := h.fetchIndex(r.Context(), repository, upstreamURL) - if index.err != nil { - h.serveIndexError(w, index.err) - return - } - - rewritten, err := h.proxy.cachedRewrite(r.Context(), helmMetadataEcosystem, h.proxyURL, cacheKey, index.body, index.digest, - func(body []byte) ([]byte, []string, error) { - return h.rewriteIndex(repository, upstreamURL, body) - }) - if err != nil { - if r.Context().Err() != nil { - return // the client left while waiting on a shared rewrite - } - h.proxy.Logger.Warn("failed to rewrite Helm index", "repository", repository, "error", err) - http.Error(w, "invalid Helm repository index", http.StatusBadGateway) - return - } - - markStale(w, index.stale) - serveRewrittenMetadataAs(w, r, rewritten, index.contentType) -} - -func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { - repository, upstreamURL, ok := h.repositoryForRequest(r) - if !ok { - http.NotFound(w, r) - return - } - - digest, ok := normalizeHelmDigest(r.PathValue("digest")) - filename := r.PathValue("filename") - if !ok || filename == "" || strings.Contains(filename, "/") || containsPathTraversal(filename) { - http.Error(w, "invalid chart request", http.StatusBadRequest) - return - } - - var charts *helmChartLookup - if h.proxy.cooldownEnabled() { - var err error - charts, err = h.downloadChartLookup(r, repository, upstreamURL) - if err != nil { - cached, cacheErr := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename) - if cacheErr == nil && cached != nil { - h.serveChart(w, r, repository, digest, filename, cached) - } else { - h.proxy.writeCooldownLookupError(w, helmMetadataEcosystem, repository, err) - } - return - } - if h.withholdChart(w, charts, digest, filename) { - return - } - } - - cached, err := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename) - if err != nil { - h.proxy.Logger.Error("failed to check Helm chart cache", "error", err) - http.Error(w, "failed to check chart cache", http.StatusInternalServerError) - return - } - if cached != nil { - h.serveChart(w, r, repository, digest, filename, cached) - return - } - - var downloadURL string - if charts == nil { - index := h.fetchIndex(r.Context(), repository, upstreamURL) - if index.err != nil { - h.serveIndexError(w, index.err) - return - } - downloadURL, err = h.findChartDownload(repository, upstreamURL, index, digest, filename) - } else { - downloadURL, err = h.findChartURL(charts, digest, filename) - } - if err != nil { - if errors.Is(err, errHelmChartNotFound) { - http.NotFound(w, r) - return - } - h.proxy.Logger.Warn("failed to read Helm index", "repository", repository, "error", err) - http.Error(w, "invalid Helm repository index", http.StatusBadGateway) - return - } - - // Verify the chart against the index digest while it is fetched, so a - // mismatch is never committed to the cache, nor streamed out complete. - result, err := h.proxy.GetOrFetchArtifactFromURLWithDigest( - r.Context(), helmMetadataEcosystem, repository, digest, filename, downloadURL, "sha256:"+digest) - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch chart") - return - } - h.serveChart(w, r, repository, digest, filename, result) -} - -func (h *HelmHandler) serveChart(w http.ResponseWriter, r *http.Request, repository, digest, filename string, result *CacheResult) { - if !strings.EqualFold(result.Artifact.Digest.Encoded(), digest) { - if result.Reader != nil { - _ = result.Reader.Close() - } - if clearErr := h.proxy.ClearCachedArtifact(helmMetadataEcosystem, repository, digest, filename); clearErr != nil { - h.proxy.Logger.Warn("failed to clear Helm chart with invalid digest", "error", clearErr) - } - http.Error(w, "chart digest verification failed", http.StatusBadGateway) - return - } - - if result.Artifact.MediaType == "" { - w.Header().Set(headerContentType, "application/gzip") - } - ServeArtifactRequest(w, r, result) -} - -func (h *HelmHandler) repositoryForRequest(r *http.Request) (name, upstreamURL string, ok bool) { - name = r.PathValue("repository") - upstreamURL, ok = h.repositories[name] - return name, upstreamURL, ok -} - -func (h *HelmHandler) fetchIndex(ctx context.Context, repository, upstreamURL string) metadataResult { - return h.proxy.fetchMetadataDocument( - ctx, - helmMetadataEcosystem, - h.indexCacheKey(repository, upstreamURL), - upstreamURL+"/"+helmIndexFilename, - "application/x-yaml, text/yaml;q=0.9, */*;q=0.1", - nil, - ) -} - -func (h *HelmHandler) indexCacheKey(repository, upstreamURL string) string { - identity := repository + "\x00" + upstreamURL - digest := sha256.Sum256([]byte(identity)) - return hex.EncodeToString(digest[:]) -} - -func (h *HelmHandler) serveIndexError(w http.ResponseWriter, err error) { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "Helm repository not found", http.StatusNotFound) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("failed to fetch Helm index", "error", err) - http.Error(w, "failed to fetch Helm repository index", http.StatusBadGateway) -} - -// rewriteIndex points the index's chart URLs at the proxy and drops releases -// in cooldown. It also returns the releases it kept, by chart name and -// digest, which together with the index fix its output. -func (h *HelmHandler) rewriteIndex(repository, upstreamURL string, body []byte) ([]byte, []string, error) { - document, entries, err := parseHelmIndex(body) - if err != nil { - return nil, nil, err - } - - var kept []string - for i := 0; i < len(entries.Content); i += 2 { - chartName := entries.Content[i].Value - releases := entries.Content[i+1] - if releases.Kind != yaml.SequenceNode { - return nil, nil, fmt.Errorf("chart %q releases must be a sequence", chartName) - } - - filtered := make([]*yaml.Node, 0, len(releases.Content)) - for _, release := range releases.Content { - chart, err := h.parseChartRelease(chartName, upstreamURL, release) - if err != nil { - h.proxy.Logger.Warn("omitting malformed Helm chart release", - "repository", repository, "chart", chartName, "error", err) - continue - } - if h.chartOnCooldown(chartName, chart.created) { - continue - } - for _, download := range chart.downloads { - if download.oci { - download.node.Value = h.ociChartProxyURL(download.url) - } else { - download.node.Value = h.chartProxyURL(repository, chart.digest, download.filename) - } - } - filtered = append(filtered, release) - kept = append(kept, chartName+"@"+chart.digest) - } - releases.Content = filtered - } - - out, err := yaml.Marshal(document) - return out, kept, err -} - -// findChartDownload returns the upstream URL of a chart in the index. Parsing -// a large index takes a while, so the charts it lists are kept per -// repository for as long as the stored index stays the same. -func (h *HelmHandler) findChartDownload(repository, upstreamURL string, index metadataResult, digest, filename string) (string, error) { - charts, err := h.chartLookup(repository, upstreamURL, index) - if err != nil { - return "", err - } - return h.findChartURL(charts, digest, filename) -} - -func (h *HelmHandler) findChartURL(charts *helmChartLookup, digest, filename string) (string, error) { - for _, chart := range charts.byDigest[digest] { - if h.chartOnCooldown(chart.name, chart.created) { - continue - } - for _, download := range chart.downloads { - if download.filename == filename { - return download.url, nil - } - } - } - return "", errHelmChartNotFound -} - -// helmChartLookup holds the HTTP chart downloads an index lists, by chart -// digest, in index order, and each release by chart name and version. -type helmChartLookup struct { - indexDigest string - byDigest map[string][]helmChartEntry - byRelease map[helmRelease]helmReleaseDownload -} - -type helmChartEntry struct { - name string - created time.Time - downloads []helmChartDownload -} - -type helmRelease struct{ chart, version string } - -// helmReleaseDownload is what helm downloads for a release: the first URL -// the index lists for it, checked against its digest. -type helmReleaseDownload struct { - digest string - created time.Time - first helmChartDownload -} - -// chartLookup returns the charts the index lists, parsing it only when the -// last lookup built for the repository came from other bytes. An index -// without a digest, read with the metadata cache off, is parsed every time. -func (h *HelmHandler) chartLookup(repository, upstreamURL string, index metadataResult) (*helmChartLookup, error) { - key := repository + "\x00" + upstreamURL - if index.digest != "" { - h.lookupMu.Lock() - cached := h.lookups[key] - h.lookupMu.Unlock() - if cached != nil && cached.indexDigest == index.digest { - return cached, nil - } - } - - lookup, err := h.buildChartLookup(upstreamURL, index.body) - if err != nil { - return nil, err - } - if index.digest != "" { - lookup.indexDigest = index.digest - h.lookupMu.Lock() - if h.lookups == nil { - h.lookups = make(map[string]*helmChartLookup) - } - h.lookups[key] = lookup - h.lookupMu.Unlock() - } - return lookup, nil -} - -func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmChartLookup, error) { - _, entries, err := parseHelmIndex(body) - if err != nil { - return nil, err - } - - lookup := &helmChartLookup{byDigest: make(map[string][]helmChartEntry), byRelease: make(map[helmRelease]helmReleaseDownload)} - for i := 0; i < len(entries.Content); i += 2 { - chartName := entries.Content[i].Value - releases := entries.Content[i+1] - if releases.Kind != yaml.SequenceNode { - return nil, fmt.Errorf("chart %q releases must be a sequence", chartName) - } - for _, release := range releases.Content { - chart, err := h.parseChartRelease(chartName, upstreamURL, release) - if err != nil { - continue - } - entry := helmChartEntry{name: chartName, created: chart.created} - if version := helmMappingValue(release, "version"); version != nil && version.Value != "" { - // helm takes the first release listed for a version. - key := helmRelease{chartName, version.Value} - if _, listed := lookup.byRelease[key]; !listed { - first := chart.downloads[0] - lookup.byRelease[key] = helmReleaseDownload{ - digest: chart.digest, - created: chart.created, - first: helmChartDownload{url: first.url, filename: first.filename, oci: first.oci}, - } - } - } - for _, download := range chart.downloads { - if !download.oci { - // Drop the YAML node, so the lookup doesn't keep the - // parsed index alive. - entry.downloads = append(entry.downloads, helmChartDownload{url: download.url, filename: download.filename}) - } - } - lookup.byDigest[chart.digest] = append(lookup.byDigest[chart.digest], entry) - } - } - return lookup, nil -} - -func (h *HelmHandler) downloadChartLookup(r *http.Request, repository, upstreamURL string) (*helmChartLookup, error) { - key := "helm\x00" + h.indexCacheKey(repository, upstreamURL) - return publishIndex(h.proxy, key, func() metadataResult { - return h.fetchIndex(r.Context(), repository, upstreamURL) - }, func(res metadataResult) (*helmChartLookup, error) { - if res.err != nil { - return nil, res.err - } - return h.chartLookup(repository, upstreamURL, res) - }) -} - -func (h *HelmHandler) withholdChart(w http.ResponseWriter, charts *helmChartLookup, digest, filename string) bool { - withheld := false - for _, chart := range charts.byDigest[digest] { - for _, download := range chart.downloads { - if download.filename != filename { - continue - } - if !h.chartOnCooldown(chart.name, chart.created) { - return false - } - withheld = true - } - } - if withheld { - http.Error(w, ErrVersionNotFound.Error(), http.StatusNotFound) - } - return withheld -} - -func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { - return !created.IsZero() && h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() && - !h.proxy.cooldown().IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created) -} - -type helmChartDownload struct { - node *yaml.Node - url string - filename string - oci bool -} - -type helmChartRelease struct { - created time.Time - digest string - downloads []helmChartDownload -} - -var errHelmChartNotFound = errors.New("chart not found in Helm index") - -func (h *HelmHandler) parseChartRelease(chartName, upstreamURL string, release *yaml.Node) (helmChartRelease, error) { - digestNode := helmMappingValue(release, "digest") - urlsNode := helmMappingValue(release, "urls") - if digestNode == nil || urlsNode == nil || urlsNode.Kind != yaml.SequenceNode || len(urlsNode.Content) == 0 { - return helmChartRelease{}, fmt.Errorf("chart %q has no digest or URLs", chartName) - } - digest, ok := normalizeHelmDigest(digestNode.Value) - if !ok { - return helmChartRelease{}, fmt.Errorf("chart %q has invalid digest", chartName) - } - - baseURL, err := url.Parse(upstreamURL + "/" + helmIndexFilename) - if err != nil { - return helmChartRelease{}, fmt.Errorf("parsing Helm repository URL: %w", err) - } - - chart := helmChartRelease{digest: digest} - if createdNode := helmMappingValue(release, "created"); createdNode != nil && createdNode.Value != "" { - chart.created, err = time.Parse(time.RFC3339Nano, createdNode.Value) - if err != nil { - return helmChartRelease{}, fmt.Errorf("chart %q has invalid creation time: %w", chartName, err) - } - } - - for _, urlNode := range urlsNode.Content { - download, err := parseHelmChartDownload(chartName, baseURL, urlNode) - if err != nil { - return helmChartRelease{}, err - } - chart.downloads = append(chart.downloads, download) - } - return chart, nil -} - -func parseHelmChartDownload(chartName string, baseURL *url.URL, node *yaml.Node) (helmChartDownload, error) { - if node.Kind != yaml.ScalarNode { - return helmChartDownload{}, fmt.Errorf("chart %q has invalid URL", chartName) - } - reference, err := url.Parse(node.Value) - if err != nil { - return helmChartDownload{}, fmt.Errorf("parsing chart %q URL: %w", chartName, err) - } - if reference.Scheme == "oci" { - if reference.Hostname() == "" || strings.Trim(reference.Path, "/") == "" || - reference.User != nil || reference.RawQuery != "" || reference.ForceQuery || reference.Fragment != "" || - containsPathTraversal(reference.Path) { - return helmChartDownload{}, fmt.Errorf("chart %q has invalid OCI reference", chartName) - } - return helmChartDownload{node: node, url: node.Value, oci: true}, nil - } - downloadURL := baseURL.ResolveReference(reference) - if (downloadURL.Scheme != "http" && downloadURL.Scheme != "https") || downloadURL.Host == "" { - return helmChartDownload{}, fmt.Errorf("chart %q URL must be HTTP(S) or OCI", chartName) - } - filename := path.Base(downloadURL.Path) - if filename == "." || filename == "/" || filename == "" || !strings.HasSuffix(filename, ".tgz") { - return helmChartDownload{}, fmt.Errorf("chart %q URL must point to a .tgz file", chartName) - } - return helmChartDownload{node: node, url: downloadURL.String(), filename: filename}, nil -} - -func (h *HelmHandler) chartProxyURL(repository, digest, filename string) string { - return fmt.Sprintf("%s/helm/%s/charts/%s/%s", h.proxyURL, - url.PathEscape(repository), digest, url.PathEscape(filename)) -} - -func parseHelmIndex(body []byte) (*yaml.Node, *yaml.Node, error) { - var document yaml.Node - if err := yaml.Unmarshal(body, &document); err != nil { - return nil, nil, fmt.Errorf("parsing Helm index: %w", err) - } - entries, err := helmIndexEntries(&document) - if err != nil { - return nil, nil, err - } - return &document, entries, nil -} - -func helmIndexEntries(document *yaml.Node) (*yaml.Node, error) { - if document == nil { - return nil, errors.New("helm index is empty") - } - if len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode { - return nil, errors.New("helm index must be a mapping") - } - entries := helmMappingValue(document.Content[0], "entries") - if entries == nil || entries.Kind != yaml.MappingNode { - return nil, errors.New("helm index has no entries mapping") - } - if len(entries.Content)%2 != 0 { - return nil, errors.New("helm index entries mapping has an incomplete key-value pair") - } - return entries, nil -} - -func helmMappingValue(mapping *yaml.Node, key string) *yaml.Node { - if mapping == nil || mapping.Kind != yaml.MappingNode { - return nil - } - for i := 0; i+1 < len(mapping.Content); i += 2 { - if mapping.Content[i].Value == key { - return mapping.Content[i+1] - } - } - return nil -} - -func normalizeHelmDigest(value string) (string, bool) { - digest := strings.TrimPrefix(strings.ToLower(value), "sha256:") - if len(digest) != sha256HexLength { - return "", false - } - for _, char := range digest { - if (char < '0' || char > '9') && (char < 'a' || char > 'f') { - return "", false - } - } - return digest, true -} diff --git a/internal/handler/helm_mirror.go b/internal/handler/helm_mirror.go deleted file mode 100644 index fe2199d..0000000 --- a/internal/handler/helm_mirror.go +++ /dev/null @@ -1,67 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "path" - "slices" - "strings" -) - -// MirrorChart caches a chart release of a configured repository under the -// key handleChart serves it from, checked against the digest the index -// lists. The index is read through the metadata cache, as handleIndex reads -// it, so with metadata caching on helm finds the chart there as well while -// the repository is down. -func (h *HelmHandler) MirrorChart(ctx context.Context, repository, chart, version string) (*CacheResult, error) { - upstreamURL, ok := h.repositories[repository] - if !ok { - return nil, h.unknownRepositoryError(repository) - } - index := h.fetchIndex(ctx, repository, upstreamURL) - if index.err != nil { - return nil, fmt.Errorf("fetching the %s index: %w", repository, index.err) - } - charts, err := h.chartLookup(repository, upstreamURL, index) - if err != nil { - return nil, err - } - - release, ok := charts.byRelease[helmRelease{chart, version}] - switch { - case !ok: - return nil, fmt.Errorf("%w: %s %s is not in the %s index", ErrUpstreamNotFound, chart, version, repository) - case release.first.oci: - return nil, fmt.Errorf("%s %s is published as %s, which helm pulls through the container registry routes; mirror it as an image: %s", - chart, version, release.first.url, ociChartPURL(release.first.url, version)) - case h.chartOnCooldown(chart, release.created): - return nil, fmt.Errorf("%s %s is in cooldown, so the index doesn't list it", chart, version) - } - return h.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, helmMetadataEcosystem, repository, release.digest, - release.first.filename, release.first.url, "sha256:"+release.digest) -} - -func (h *HelmHandler) unknownRepositoryError(repository string) error { - names := make([]string, 0, len(h.repositories)) - for name := range h.repositories { - names = append(names, name) - } - if len(names) == 0 { - return fmt.Errorf("unknown Helm repository %q: upstream.helm configures none", repository) - } - slices.Sort(names) - return fmt.Errorf("unknown Helm repository %q: upstream.helm configures %s", repository, strings.Join(names, ", ")) -} - -// ociChartPURL returns the image PURL that mirrors an OCI chart reference, -// tagged with the chart version when the reference has no tag or digest, as -// helm pulls it. -func ociChartPURL(reference, version string) string { - repository := strings.TrimPrefix(reference, "oci://") - if at := strings.LastIndex(repository, "@"); at >= 0 { - repository, version = repository[:at], repository[at+1:] - } else if colon := strings.LastIndex(repository, ":"); colon > strings.LastIndex(repository, "/") { - repository, version = repository[:colon], repository[colon+1:] - } - return "pkg:oci/" + path.Base(repository) + "@" + version + "?repository_url=" + repository -} diff --git a/internal/handler/helm_mirror_test.go b/internal/handler/helm_mirror_test.go deleted file mode 100644 index 617d35d..0000000 --- a/internal/handler/helm_mirror_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package handler - -import "testing" - -func TestOCIChartPURL(t *testing.T) { - tests := []struct{ reference, want string }{ - {"oci://ghcr.io/owner/demo:2.0.0", "pkg:oci/demo@2.0.0?repository_url=ghcr.io/owner/demo"}, - {"oci://ghcr.io/owner/demo", "pkg:oci/demo@1.0.0?repository_url=ghcr.io/owner/demo"}, - {"oci://localhost:5000/demo", "pkg:oci/demo@1.0.0?repository_url=localhost:5000/demo"}, - {"oci://ghcr.io/owner/demo@sha256:abc", "pkg:oci/demo@sha256:abc?repository_url=ghcr.io/owner/demo"}, - } - for _, tt := range tests { - if got := ociChartPURL(tt.reference, "1.0.0"); got != tt.want { - t.Errorf("ociChartPURL(%q) = %q, want %q", tt.reference, got, tt.want) - } - } -} diff --git a/internal/handler/helm_oci.go b/internal/handler/helm_oci.go deleted file mode 100644 index 00392e8..0000000 --- a/internal/handler/helm_oci.go +++ /dev/null @@ -1,64 +0,0 @@ -package handler - -import ( - "net/url" - "slices" - "strings" -) - -type helmOCIRegistry struct { - host string - prefix string -} - -// NewHelmHandlerWithOCIRegistries also rewrites OCI references for configured -// registries through the existing /v2 routes. Unmatched references stay intact. -func NewHelmHandlerWithOCIRegistries(proxy *Proxy, proxyURL string, repositories map[string]string, defaultRegistry string, registries map[string]string) *HelmHandler { - h := NewHelmHandler(proxy, proxyURL, repositories) - // Prefer named routes; use a stable order when aliases share a registry. - names := make([]string, 0, len(registries)) - for name := range registries { - names = append(names, name) - } - slices.Sort(names) - for _, name := range names { - h.addOCIRegistry(registries[name], "/upstream/"+url.PathEscape(name)) - } - h.addOCIRegistry(configuredUpstreamURL(defaultRegistry, dockerHubRegistry), "") - return h -} - -func (h *HelmHandler) addOCIRegistry(rawURL, prefix string) { - u, err := url.Parse(rawURL) - // A path-prefixed upstream is an API mount, not an OCI repository prefix; - // its mapping cannot be inferred from a chart's registry authority alone. - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || - strings.Trim(u.Path, "/") != "" || u.RawQuery != "" || u.Fragment != "" || u.User != nil { - return - } - h.ociRegistries = append(h.ociRegistries, helmOCIRegistry{host: u.Host, prefix: prefix}) -} - -func (h *HelmHandler) ociChartProxyURL(reference string) string { - u, err := url.Parse(reference) - if err != nil { - return reference - } - proxy, err := url.Parse(h.proxyURL) - // OCI clients place /v2 before the repository path, so a path-prefixed - // public HTTP base URL cannot be represented by simply prepending its path. - if err != nil || proxy.Host == "" || strings.Trim(proxy.Path, "/") != "" { - return reference - } - for _, registry := range h.ociRegistries { - if !strings.EqualFold(u.Host, registry.host) { - continue - } - if registry.prefix == "" && strings.HasPrefix(u.Path, "/upstream/") { - // This prefix is reserved by the named-registry router. - return reference - } - return "oci://" + proxy.Host + registry.prefix + u.EscapedPath() - } - return reference -} diff --git a/internal/handler/helm_oci_test.go b/internal/handler/helm_oci_test.go deleted file mode 100644 index e421e3f..0000000 --- a/internal/handler/helm_oci_test.go +++ /dev/null @@ -1,216 +0,0 @@ -package handler - -import ( - "errors" - "fmt" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/registries/fetch" -) - -func TestHelmHandler_MixedOCIIndex(t *testing.T) { - chart := "http chart content" - digest := helmSHA256Hex([]byte(chart)) - const ociURL = "oci://ghcr.io/stakater/saap-catalog/charts/konfigurator-0.1.40.tgz" - index := fmt.Sprintf(`apiVersion: v1 -entries: - konfigurator: - - digest: %s - urls: [%s] - version: 0.1.40 - demo: - - digest: %s - urls: [oci://ghcr.io/owner/demo:1.0.0, demo-1.0.0.tgz] - version: 1.0.0 -`, digest, ociURL, digest) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/index.yaml": - _, _ = fmt.Fprint(w, index) - case "/demo-1.0.0.tgz": - _, _ = fmt.Fprint(w, chart) - default: - t.Errorf("unexpected HTTP upstream request: %s", r.URL.Path) - http.NotFound(w, r) - } - })) - defer upstream.Close() - p, _, _, _ := setupTestProxy(t) - p.CacheMetadata = true - p.MetadataTTL = time.Hour - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - p.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewHelmHandler(p, "https://proxy.example", map[string]string{"mixed": upstream.URL}) - response := serveHelmRequest(h, "/mixed/index.yaml") - if response.Code != http.StatusOK { - t.Fatalf("index status = %d: %s", response.Code, response.Body.String()) - } - if !strings.Contains(response.Body.String(), ociURL) || !strings.Contains(response.Body.String(), h.chartProxyURL("mixed", digest, "demo-1.0.0.tgz")) { - t.Fatalf("incorrectly rewritten index: %s", response.Body.String()) - } - download, err := h.findChartDownload("mixed", upstream.URL, metadataResult{body: []byte(index)}, digest, "demo-1.0.0.tgz") - if err != nil || download != upstream.URL+"/demo-1.0.0.tgz" { - t.Fatalf("HTTP chart lookup = %q, %v", download, err) - } - // Even a .tgz-shaped OCI reference must never reach the HTTP fetcher. - response = serveHelmRequest(h, "/mixed/charts/"+digest+"/konfigurator-0.1.40.tgz") - if response.Code != http.StatusNotFound { - t.Fatalf("OCI reference reached HTTP download route: status=%d", response.Code) - } - response = serveHelmRequest(h, "/mixed/charts/"+digest+"/demo-1.0.0.tgz") - if response.Code != http.StatusOK || response.Body.String() != chart { - t.Fatalf("cold HTTP chart status=%d body=%s", response.Code, response.Body.String()) - } - upstream.Close() - response = serveHelmRequest(h, "/mixed/charts/"+digest+"/demo-1.0.0.tgz") - if response.Code != http.StatusOK || response.Body.String() != chart { - t.Fatalf("cached HTTP chart status=%d body=%s", response.Code, response.Body.String()) - } -} - -func TestHelmOCIReferenceRewriting(t *testing.T) { - for _, tt := range []struct { - name, reference, defaultRegistry, proxyURL string - registries map[string]string - want string - }{ - {name: "unmatched", reference: "oci://ghcr.io/owner/chart:1.0.0", want: "oci://ghcr.io/owner/chart:1.0.0"}, - {name: "named", reference: "oci://ghcr.io/owner/chart:1.0.0", registries: map[string]string{"ghcr": "https://ghcr.io"}, want: "oci://proxy.example/upstream/ghcr/owner/chart:1.0.0"}, - {name: "digest", reference: "oci://ghcr.io/owner/chart@sha256:" + strings.Repeat("a", 64), registries: map[string]string{"ghcr": "https://ghcr.io/"}, want: "oci://proxy.example/upstream/ghcr/owner/chart@sha256:" + strings.Repeat("a", 64)}, - {name: "reported tgz reference", reference: "oci://ghcr.io/stakater/saap-catalog/charts/konfigurator-0.1.40.tgz", registries: map[string]string{"ghcr": "https://ghcr.io"}, want: "oci://proxy.example/upstream/ghcr/stakater/saap-catalog/charts/konfigurator-0.1.40.tgz"}, - {name: "default", reference: "oci://registry.example/owner/chart:1.0.0", defaultRegistry: "https://registry.example", want: "oci://proxy.example/owner/chart:1.0.0"}, - {name: "named preferred and stable", reference: "oci://ghcr.io/owner/chart", defaultRegistry: "https://ghcr.io", registries: map[string]string{"z": "https://ghcr.io", "a": "https://ghcr.io"}, want: "oci://proxy.example/upstream/a/owner/chart"}, - {name: "port", reference: "oci://registry.example:5000/owner/chart", registries: map[string]string{"local": "http://registry.example:5000"}, proxyURL: "http://localhost:8080", want: "oci://localhost:8080/upstream/local/owner/chart"}, - {name: "different port", reference: "oci://registry.example:5001/owner/chart", registries: map[string]string{"local": "http://registry.example:5000"}, want: "oci://registry.example:5001/owner/chart"}, - {name: "different host", reference: "oci://ghcr.io.evil.example/owner/chart", registries: map[string]string{"ghcr": "https://ghcr.io"}, want: "oci://ghcr.io.evil.example/owner/chart"}, - {name: "escaped path", reference: "oci://ghcr.io/owner/chart:1.0.0%2Bbuild", registries: map[string]string{"ghcr": "https://ghcr.io"}, want: "oci://proxy.example/upstream/ghcr/owner/chart:1.0.0%2Bbuild"}, - {name: "upstream API mount", reference: "oci://ghcr.io/owner/chart", registries: map[string]string{"ghcr": "https://ghcr.io/mount"}, want: "oci://ghcr.io/owner/chart"}, - {name: "proxy path prefix", reference: "oci://ghcr.io/owner/chart", proxyURL: "https://proxy.example/mount", registries: map[string]string{"ghcr": "https://ghcr.io"}, want: "oci://ghcr.io/owner/chart"}, - {name: "reserved default prefix", reference: "oci://ghcr.io/upstream/other/chart", defaultRegistry: "https://ghcr.io", want: "oci://ghcr.io/upstream/other/chart"}, - } { - t.Run(tt.name, func(t *testing.T) { - if tt.proxyURL == "" { - tt.proxyURL = "https://proxy.example" - } - h := NewHelmHandlerWithOCIRegistries(&Proxy{}, tt.proxyURL, nil, tt.defaultRegistry, tt.registries) - index := fmt.Sprintf("apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [%q]\n", strings.Repeat("a", 64), tt.reference) - rewritten, _, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) - if err != nil { - t.Fatal(err) - } - _, entries, err := parseHelmIndex(rewritten) - if err != nil { - t.Fatal(err) - } - got := helmMappingValue(entries.Content[1].Content[0], "urls").Content[0].Value - if got != tt.want { - t.Fatalf("rewritten reference = %q, want %q", got, tt.want) - } - }) - } -} - -func TestHelmIndexOmitsMalformedChartReleases(t *testing.T) { - goodDigest := strings.Repeat("a", 64) - for _, reference := range []string{"oci:///chart", "oci://ghcr.io", "oci://user:secret@ghcr.io/chart", "oci://ghcr.io/../chart", "oci://ghcr.io/%2e%2e/chart", "oci://ghcr.io/chart?token=x", "file:///chart.tgz", "ftp://example.com/chart.tgz", "https://example.com/chart.zip"} { - t.Run(reference, func(t *testing.T) { - h := NewHelmHandler(&Proxy{Logger: slog.New(slog.NewTextHandler(io.Discard, nil))}, "https://proxy.example", nil) - index := fmt.Sprintf("apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [%q]\n - digest: %s\n urls: [demo-1.0.0.tgz]\n", strings.Repeat("b", 64), reference, goodDigest) - rewritten, _, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(rewritten), reference) || strings.Contains(string(rewritten), strings.Repeat("b", 64)) { - t.Fatalf("malformed release not omitted: %s", rewritten) - } - if !strings.Contains(string(rewritten), h.chartProxyURL("mixed", goodDigest, "demo-1.0.0.tgz")) { - t.Fatalf("valid release missing from rewritten index: %s", rewritten) - } - download, err := h.findChartDownload("mixed", "https://charts.example", metadataResult{body: []byte(index)}, goodDigest, "demo-1.0.0.tgz") - if err != nil || download != "https://charts.example/demo-1.0.0.tgz" { - t.Fatalf("chart lookup = %q, %v", download, err) - } - if _, err := h.findChartDownload("mixed", "https://charts.example", metadataResult{body: []byte(index)}, strings.Repeat("b", 64), "demo-1.0.0.tgz"); !errors.Is(err, errHelmChartNotFound) { - t.Fatalf("omitted release lookup error = %v, want errHelmChartNotFound", err) - } - }) - } -} - -func TestHelmOCICooldown(t *testing.T) { - p := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}} - h := NewHelmHandlerWithOCIRegistries(p, "https://proxy.example", nil, "https://ghcr.io", nil) - index := fmt.Sprintf(`apiVersion: v1 -entries: - demo: - - digest: %s - created: %s - urls: [oci://ghcr.io/owner/demo:1.0.0] - - digest: %s - created: %s - urls: [oci://ghcr.io/owner/demo:2.0.0] -`, strings.Repeat("a", 64), time.Now().Add(-7*24*time.Hour).Format(time.RFC3339), - strings.Repeat("b", 64), time.Now().Add(-time.Hour).Format(time.RFC3339)) - rewritten, _, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(rewritten), "demo:2.0.0") || !strings.Contains(string(rewritten), "oci://proxy.example/owner/demo:1.0.0") { - t.Fatalf("incorrect cooldown filtering: %s", rewritten) - } -} - -func TestHelmRewrittenOCIRouteServesChart(t *testing.T) { - chart := "OCI Helm chart layer" - digest := "sha256:" + helmSHA256Hex([]byte(chart)) - manifest := `{"schemaVersion":2,"layers":[{"mediaType":"application/vnd.cncf.helm.chart.content.v1.tar+gzip","digest":"` + digest + `"}]}` - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/v2/owner/demo/manifests/1.0.0": - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - _, _ = fmt.Fprint(w, manifest) - case "/v2/owner/demo/blobs/" + digest: - _, _ = fmt.Fprint(w, chart) - default: - t.Errorf("unexpected registry request: %s", r.URL.Path) - http.NotFound(w, r) - } - })) - defer upstream.Close() - p, _, _, _ := setupTestProxy(t) - p.HTTPClient = upstream.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - p.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - registries := map[string]string{"local": upstream.URL} - h := NewHelmHandlerWithOCIRegistries(p, "https://proxy.example", nil, "", registries) - reference := "oci://" + strings.TrimPrefix(upstream.URL, "http://") + "/owner/demo:1.0.0" - rewritten, err := url.Parse(h.ociChartProxyURL(reference)) - if err != nil { - t.Fatal(err) - } - if rewritten.Host != "proxy.example" { - t.Fatalf("OCI reference was not proxied: %s", rewritten) - } - repository := strings.TrimSuffix(rewritten.Path, ":1.0.0") - container := NewContainerHandlerWithRegistry(p, "https://proxy.example", "", registries) - for _, tt := range []struct{ path, want string }{ - {repository + "/manifests/1.0.0", manifest}, - {repository + "/blobs/" + digest, chart}, - } { - w := httptest.NewRecorder() - container.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, tt.path, nil)) - if w.Code != http.StatusOK || w.Body.String() != tt.want { - t.Fatalf("OCI GET %s: status=%d body=%s", tt.path, w.Code, w.Body.String()) - } - } -} diff --git a/internal/handler/helm_pypi_bench_test.go b/internal/handler/helm_pypi_bench_test.go deleted file mode 100644 index afda74f..0000000 --- a/internal/handler/helm_pypi_bench_test.go +++ /dev/null @@ -1,184 +0,0 @@ -package handler - -import ( - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" -) - -// benchHelmIndex builds an index.yaml shaped like a large public chart -// repository: many charts, each with many releases. -func benchHelmIndex(charts, releases int) string { - var b strings.Builder - b.WriteString("apiVersion: v1\nentries:\n") - for c := range charts { - fmt.Fprintf(&b, " chart-%d:\n", c) - for r := range releases { - fmt.Fprintf(&b, ` - apiVersion: v2 - appVersion: 1.%d.0 - created: "2021-01-02T03:04:05.123456789Z" - dependencies: - - name: common - repository: oci://registry.example/charts - version: 2.x.x - description: A chart for benchmarking the index rewrite with a realistic amount of text. - digest: %064x - home: https://example.com/chart-%d - keywords: [web, server, benchmark] - maintainers: - - name: Maintainer - url: https://example.com - name: chart-%d - sources: - - https://github.com/example/chart-%d - urls: - - https://charts.example/chart-%d-1.%d.0.tgz - version: 1.%d.0 -`, r, c*releases+r, c, c, c, c, r, r) - } - } - b.WriteString("generated: \"2021-01-02T03:04:05Z\"\n") - return b.String() -} - -func benchServe(b *testing.B, h http.Handler, target, accept string) { - b.Helper() - b.ReportAllocs() - for b.Loop() { - rec := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, target, nil) - if accept != "" { - req.Header.Set("Accept", accept) - } - h.ServeHTTP(rec, req) - if rec.Code != http.StatusOK { - b.Fatalf("status %d: %s", rec.Code, rec.Body.String()) - } - } -} - -// BenchmarkHelmIndexRequest serves a cached 2.5 MB index repeatedly. -func BenchmarkHelmIndexRequest(b *testing.B) { - index := benchHelmIndex(250, 20) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/x-yaml") - _, _ = w.Write([]byte(index)) - })) - defer upstream.Close() - proxy, _, _, _ := setupTestProxy(b) - proxy.HTTPClient = upstream.Client() - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.SetMetadataRewriteCacheSize(256 << 20) - h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": upstream.URL}).Routes() - b.SetBytes(int64(len(index))) - benchServe(b, h, "/stable/index.yaml", "") -} - -// benchPyPIProject builds simple HTML, simple JSON and JSON API documents -// for a project with many files, like botocore. -func benchPyPIProject(versions, filesPerVersion int) (simpleHTML, simpleJSON, jsonAPI string) { - var hb, sb, jb strings.Builder - hb.WriteString("\n\n \n \n Links for big\n \n \n

Links for big

\n") - sb.WriteString(`{"meta":{"api-version":"1.1"},"name":"big","files":[`) - jb.WriteString(`{"info":{"name":"big","version":"1.0.0"},"releases":{`) - n := 0 - for v := range versions { - version := fmt.Sprintf("1.%d.0", v) - if v > 0 { - jb.WriteByte(',') - } - fmt.Fprintf(&jb, "%q:[", version) - for f := range filesPerVersion { - filename := fmt.Sprintf("big-%s-py3-none-any%d.whl", version, f) - if f == 1 { - filename = fmt.Sprintf("big-%s.tar.gz", version) - } - url := fmt.Sprintf("https://files.pythonhosted.org/packages/%02x/%02x/%060x/%s", v%256, f, v*filesPerVersion+f, filename) - fmt.Fprintf(&hb, " %s
\n", url, n, n, n, filename) - if n > 0 { - sb.WriteByte(',') - } - fmt.Fprintf(&sb, `{"filename":%q,"url":%q,"hashes":{"sha256":"%064x"},"requires-python":">=3.8","core-metadata":{"sha256":"%064x"},"size":12345,"upload-time":"2021-01-02T03:04:05.123456Z","yanked":false}`, filename, url, n, n) - if f > 0 { - jb.WriteByte(',') - } - fmt.Fprintf(&jb, `{"filename":%q,"url":%q,"digests":{"sha256":"%064x","md5":"%032x"},"packagetype":"bdist_wheel","python_version":"py3","size":12345,"upload_time_iso_8601":"2021-01-02T03:04:05.123456Z","yanked":false}`, filename, url, n, n) - n++ - } - jb.WriteByte(']') - } - hb.WriteString(" \n\n\n") - sb.WriteString("]}") - jb.WriteString(`},"urls":[]}`) - return hb.String(), sb.String(), jb.String() -} - -func benchPyPIHandler(b *testing.B, deny ...string) http.Handler { - b.Helper() - simpleHTML, simpleJSON, jsonAPI := benchPyPIProject(2000, 2) - h, proxy := setupPyPIHandler(b, func(r *http.Request) (*http.Response, error) { - switch { - case r.URL.Path == "/pypi/big/json": - return pypiHTTPResponse(r, "application/json", jsonAPI), nil - case r.Header.Get("Accept") == pypiSimpleJSON: - return pypiHTTPResponse(r, pypiSimpleJSON, simpleJSON), nil - default: - return pypiHTTPResponse(r, "text/html", simpleHTML), nil - } - }) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.SetMetadataRewriteCacheSize(256 << 20) - if len(deny) > 0 { - setTestDenylist(b, proxy, deny...) - } - return h.Routes() -} - -func BenchmarkPyPISimpleHTMLRequest(b *testing.B) { - benchServe(b, benchPyPIHandler(b), "/simple/big/", "") -} - -// BenchmarkPyPISimpleHTMLFilteredRequest has a version denied, so the page -// is filtered as well as rewritten. -func BenchmarkPyPISimpleHTMLFilteredRequest(b *testing.B) { - benchServe(b, benchPyPIHandler(b, "pkg:pypi/big@1.5.0"), "/simple/big/", "") -} - -func BenchmarkPyPISimpleJSONRequest(b *testing.B) { - benchServe(b, benchPyPIHandler(b), "/simple/big/", pypiSimpleJSON) -} - -func BenchmarkPyPIJSONAPIRequest(b *testing.B) { - benchServe(b, benchPyPIHandler(b), "/pypi/big/json", "") -} - -// BenchmarkPyPIRewriteSimpleHTML is the rewrite a cache miss pays. -func BenchmarkPyPIRewriteSimpleHTML(b *testing.B) { - simpleHTML, _, _ := benchPyPIProject(2000, 2) - body := []byte(simpleHTML) - h := NewPyPIHandlerWithUpstreams(testProxy(), "http://proxy.example", "", "") - b.SetBytes(int64(len(body))) - b.ReportAllocs() - for b.Loop() { - _ = h.rewriteSimpleHTML(body, nil) - } -} - -// BenchmarkPyPIRewriteSimpleHTMLFiltered is the same with a version filtered -// out. -func BenchmarkPyPIRewriteSimpleHTMLFiltered(b *testing.B) { - simpleHTML, _, _ := benchPyPIProject(2000, 2) - body := []byte(simpleHTML) - h := NewPyPIHandlerWithUpstreams(testProxy(), "http://proxy.example", "", "") - filtered := map[string]bool{"1.5.0": true} - b.SetBytes(int64(len(body))) - b.ReportAllocs() - for b.Loop() { - _ = h.rewriteSimpleHTML(h.filterSimpleHTMLLinks(body, filtered), filtered) - } -} diff --git a/internal/handler/helm_pypi_rewrite_test.go b/internal/handler/helm_pypi_rewrite_test.go deleted file mode 100644 index 3500f8e..0000000 --- a/internal/handler/helm_pypi_rewrite_test.go +++ /dev/null @@ -1,308 +0,0 @@ -package handler - -import ( - "context" - "errors" - "fmt" - "net/http" - "net/http/httptest" - "regexp" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// getRewritten requests target and checks the response status. -func getRewritten(t *testing.T, h http.Handler, target string, header http.Header, want int) *httptest.ResponseRecorder { - t.Helper() - req := httptest.NewRequest(http.MethodGet, target, nil) - for k, v := range header { - req.Header[k] = v - } - rec := httptest.NewRecorder() - h.ServeHTTP(rec, req) - if rec.Code != want { - t.Fatalf("GET %s: status %d, want %d: %s", target, rec.Code, want, rec.Body.String()) - } - return rec -} - -// checkServedFromRewriteCache requests target twice more after a first -// request, with storage failing, and checks the repeat is the cached rewrite: -// same body and ETag, 304 for the ETag, gzip for a client accepting it. -func checkServedFromRewriteCache(t *testing.T, h http.Handler, store *mockStorage, target string, header http.Header) string { - t.Helper() - first := getRewritten(t, h, target, header, http.StatusOK) - etag := first.Header().Get(headerETag) - if etag == "" { - t.Fatalf("no ETag on rewritten metadata; headers %v", first.Header()) - } - - // The repeat must not read the stored document back. - store.mu.Lock() - store.openErr = errors.New("storage must not be read") - store.mu.Unlock() - t.Cleanup(func() { - store.mu.Lock() - store.openErr = nil - store.mu.Unlock() - }) - - second := getRewritten(t, h, target, header, http.StatusOK) - if second.Body.String() != first.Body.String() || second.Header().Get(headerETag) != etag { - t.Errorf("repeat differs: ETag %q vs %q", second.Header().Get(headerETag), etag) - } - if got, want := second.Header().Get(headerContentType), first.Header().Get(headerContentType); got != want { - t.Errorf("repeat Content-Type = %q, want %q", got, want) - } - - conditional := header.Clone() - if conditional == nil { - conditional = http.Header{} - } - conditional.Set("If-None-Match", etag) - getRewritten(t, h, target, conditional, http.StatusNotModified) - - gzipped := header.Clone() - if gzipped == nil { - gzipped = http.Header{} - } - gzipped.Set(headerAcceptEncoding, "gzip") - compressed := getRewritten(t, h, target, gzipped, http.StatusOK) - if got := compressed.Header().Get(headerContentEncoding); got != "gzip" { - t.Errorf("Content-Encoding = %q, want gzip", got) - } - return first.Body.String() -} - -func TestHelmIndexServedFromRewriteCache(t *testing.T) { - digest := strings.Repeat("a", 64) - var indexRequests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - indexRequests.Add(1) - w.Header().Set("Content-Type", "application/x-yaml") - _, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n%s", strings.Repeat(fmt.Sprintf( - " - digest: %s\n name: demo\n urls: [demo-1.0.0.tgz]\n version: 1.0.0\n", digest), 20)) - })) - defer upstream.Close() - - proxy, _, store, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.SetMetadataRewriteCacheSize(1 << 20) - h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": upstream.URL}).Routes() - - body := checkServedFromRewriteCache(t, h, store, "/stable/index.yaml", nil) - if !strings.Contains(body, "http://proxy.example/helm/stable/charts/"+digest+"/demo-1.0.0.tgz") { - t.Errorf("index not rewritten: %s", body) - } - if got := indexRequests.Load(); got != 1 { - t.Errorf("upstream requests = %d, want 1", got) - } - if got := proxy.rewrites.len(); got != 1 { - t.Errorf("cached rewrites = %d, want 1", got) - } -} - -// TestHelmChartLookupFollowsIndex checks that chart downloads are looked up -// in charts kept from the last parse of the same index, and that a changed -// index is parsed again. -func TestHelmChartLookupFollowsIndex(t *testing.T) { - chartA, chartB := []byte("chart a"), []byte("chart b") - digestA, digestB := helmSHA256Hex(chartA), helmSHA256Hex(chartB) - var withB atomic.Bool - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/index.yaml": - w.Header().Set("Content-Type", "application/x-yaml") - _, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n a:\n - digest: %s\n urls: [a-1.0.0.tgz]\n", digestA) - if withB.Load() { - _, _ = fmt.Fprintf(w, " b:\n - digest: %s\n urls: [b-1.0.0.tgz]\n", digestB) - } - case "/a-1.0.0.tgz": - _, _ = w.Write(chartA) - case "/b-1.0.0.tgz": - _, _ = w.Write(chartB) - default: - http.NotFound(w, r) - } - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": upstream.URL}) - index := func() metadataResult { - return h.fetchIndex(context.Background(), "stable", upstream.URL) - } - - first, err := h.findChartDownload("stable", upstream.URL, index(), digestA, "a-1.0.0.tgz") - if err != nil || first != upstream.URL+"/a-1.0.0.tgz" { - t.Fatalf("lookup = %q, %v", first, err) - } - lookup := h.lookups["stable\x00"+upstream.URL] - if lookup == nil { - t.Fatal("chart lookup was not kept") - } - if _, err := h.findChartDownload("stable", upstream.URL, index(), digestB, "b-1.0.0.tgz"); !errors.Is(err, errHelmChartNotFound) { - t.Fatalf("lookup of unlisted chart: %v", err) - } - if h.lookups["stable\x00"+upstream.URL] != lookup { - t.Error("same index was parsed again") - } - - // A new index from upstream lists a new chart. - withB.Store(true) - proxy.MetadataTTL = 0 - got, err := h.findChartDownload("stable", upstream.URL, index(), digestB, "b-1.0.0.tgz") - if err != nil || got != upstream.URL+"/b-1.0.0.tgz" { - t.Fatalf("lookup in new index = %q, %v", got, err) - } -} - -func TestPyPIMetadataServedFromRewriteCache(t *testing.T) { - const files = "https://files.pythonhosted.org/packages/ab/cd/" - simpleHTML := `demo-1.0.0.tar.gz
` + - `demo-2.0.0.tar.gz
` + - strings.Repeat("\n", 20) - simpleJSON := `{"meta":{"api-version":"1.1"},"name":"demo","files":[` + - `{"filename":"demo-1.0.0.tar.gz","url":"` + files + `demo-1.0.0.tar.gz"},` + - `{"filename":"demo-2.0.0.tar.gz","url":"` + files + `demo-2.0.0.tar.gz"}],` + - `"padding":"` + strings.Repeat("x", 1000) + `"}` - jsonAPI := `{"info":{"name":"demo","version":"1.0.0","description":"` + strings.Repeat("x", 1000) + `"},` + - `"releases":{"1.0.0":[{"url":"` + files + `demo-1.0.0.tar.gz"}],"2.0.0":[{"url":"` + files + `demo-2.0.0.tar.gz"}]},` + - `"urls":[{"url":"` + files + `demo-1.0.0.tar.gz"}]}` - - cases := []struct { - name, target, accept string - }{ - {"simple html", "/simple/demo/", ""}, - {"simple json", "/simple/demo/", pypiSimpleJSON}, - {"json api", "/pypi/demo/json", ""}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - var upstreamCalls atomic.Int32 - proxy, _, store, _ := setupTestProxy(t) - proxy.HTTPClient = &http.Client{Transport: pypiRoundTripFunc(func(r *http.Request) (*http.Response, error) { - upstreamCalls.Add(1) - switch { - case r.URL.Path == "/pypi/demo/json": - return pypiHTTPResponse(r, "application/json", jsonAPI), nil - case r.Header.Get("Accept") == pypiSimpleJSON: - return pypiHTTPResponse(r, pypiSimpleJSON, simpleJSON), nil - default: - return pypiHTTPResponse(r, "text/html", simpleHTML), nil - } - })} - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.SetMetadataRewriteCacheSize(1 << 20) - setTestDenylist(t, proxy, "pkg:pypi/demo@2.0.0") - h := NewPyPIHandlerWithUpstreams(proxy, "http://proxy.test", "https://pypi.test", "") - - var header http.Header - if c.accept != "" { - header = http.Header{"Accept": {c.accept}} - } - body := checkServedFromRewriteCache(t, h.Routes(), store, c.target, header) - if !strings.Contains(body, "http://proxy.test/pypi/packages/packages/ab/cd/demo-1.0.0.tar.gz") { - t.Errorf("download URL not rewritten: %s", body) - } - if strings.Contains(body, "demo-2.0.0") { - t.Errorf("denied version served: %s", body) - } - if got := upstreamCalls.Load(); got != 1 { - t.Errorf("upstream calls = %d, want 1", got) - } - - if c.target == "/simple/demo/" { - rec := getRewritten(t, h.Routes(), c.target, header, http.StatusOK) - vary := strings.Join(rec.Header().Values("Vary"), ", ") - if !strings.Contains(vary, "Accept-Encoding") || !strings.Contains(strings.ReplaceAll(vary, "Accept-Encoding", ""), "Accept") { - t.Errorf("Vary = %q, want Accept and Accept-Encoding", vary) - } - } - }) - } -} - -// TestPyPISimpleRewriteFollowsCooldown checks that a cached rewrite of a -// simple page is not served once the versions cooldown filters out change, -// though the page itself is the same. -func TestPyPISimpleRewriteFollowsCooldown(t *testing.T) { - now := time.Now() - old := now.Add(-30 * 24 * time.Hour).Format(time.RFC3339) - var released atomic.Value - released.Store(now.Add(-time.Hour).Format(time.RFC3339)) - - h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - switch r.URL.Path { - case "/simple/ruff/": - return pypiHTTPResponse(r, "text/html", - `ruff-1.0.0.tar.gz`+ - `ruff-2.0.0.tar.gz`), nil - case "/pypi/ruff/json": - return pypiHTTPResponse(r, "application/json", `{"releases":{`+ - `"1.0.0":[{"upload_time_iso_8601":"`+old+`"}],`+ - `"2.0.0":[{"upload_time_iso_8601":"`+released.Load().(string)+`"}]}}`), nil - default: - t.Fatalf("unexpected upstream path: %s", r.URL.Path) - return nil, nil - } - }) - proxy.Cooldown = &cooldown.Config{Default: "7d"} - proxy.SetMetadataRewriteCacheSize(1 << 20) - - first := getRewritten(t, h.Routes(), "/simple/ruff/", nil, http.StatusOK).Body.String() - if strings.Contains(first, "ruff-2.0.0") || !strings.Contains(first, "ruff-1.0.0") { - t.Fatalf("first page = %s, want only ruff-1.0.0", first) - } - - released.Store(old) - second := getRewritten(t, h.Routes(), "/simple/ruff/", nil, http.StatusOK).Body.String() - if !strings.Contains(second, "ruff-2.0.0") { - t.Errorf("version out of cooldown still filtered: %s", second) - } -} - -// rewriteDownloadHrefsRegexp is the regular expression rewrite that -// rewriteDownloadHrefs replaces. -func rewriteDownloadHrefsRegexp(h *PyPIHandler, body []byte) []byte { - re := regexp.MustCompile(`href="(` + regexp.QuoteMeta(h.downloadURL) + `/packages/[^"]+)"`) - return re.ReplaceAllFunc(body, func(match []byte) []byte { - submatch := re.FindSubmatch(match) - return []byte(`href="` + h.proxyURL + "/pypi/packages" + strings.TrimPrefix(string(submatch[1]), h.downloadURL) + `"`) - }) -} - -func FuzzPyPIRewriteDownloadHrefs(f *testing.F) { - const dl = "https://files.pythonhosted.org" - for _, seed := range []string{ - `x`, - `empty`, - ``, - ``, - `x`, - `x`, - `x`, - `href="` + dl + `/packages/` + "\n" + `multi line"`, - `href="href="` + dl + `/packages/x"`, - } { - f.Add(seed) - } - h := NewPyPIHandlerWithUpstreams(testProxy(), "http://proxy.test", "", dl) - f.Fuzz(func(t *testing.T, body string) { - want := rewriteDownloadHrefsRegexp(h, []byte(body)) - if got := h.rewriteDownloadHrefs([]byte(body)); string(got) != string(want) { - t.Errorf("rewriteDownloadHrefs(%q)\n = %q\nwant %q", body, got, want) - } - }) -} diff --git a/internal/handler/helm_test.go b/internal/handler/helm_test.go deleted file mode 100644 index 837934a..0000000 --- a/internal/handler/helm_test.go +++ /dev/null @@ -1,349 +0,0 @@ -package handler - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "net/http" - "net/http/httptest" - "slices" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" - "github.com/git-pkgs/registries/fetch" - "gopkg.in/yaml.v3" -) - -func TestHelmHandler_RewritesIndexAndCachesChart(t *testing.T) { - chart := []byte("a Helm chart") - digest := helmSHA256Hex(chart) - var available atomic.Bool - available.Store(true) - var indexRequests atomic.Int32 - var chartRequests atomic.Int32 - - var upstream *httptest.Server - upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - switch r.URL.Path { - case "/charts/index.yaml": - indexRequests.Add(1) - w.Header().Set("Content-Type", "application/x-yaml") - _, _ = fmt.Fprintf(w, `apiVersion: v1 -entries: - demo: - - annotations: - example.com/retained: "true" - created: 2020-01-02T03:04:05Z - digest: %s - name: demo - urls: - - demo-1.0.0.tgz - - %s/charts/mirror/demo-1.0.0.tgz - version: 1.0.0 -generated: 2020-01-02T03:04:05Z -`, digest, upstream.URL) - case "/charts/demo-1.0.0.tgz", "/charts/mirror/demo-1.0.0.tgz": - chartRequests.Add(1) - w.Header().Set("Content-Type", "application/gzip") - _, _ = w.Write(chart) - default: - http.NotFound(w, r) - } - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": upstream.URL + "/charts"}) - - indexResponse := serveHelmRequest(h, "/stable/index.yaml") - if indexResponse.Code != http.StatusOK { - t.Fatalf("index status = %d, want 200: %s", indexResponse.Code, indexResponse.Body.String()) - } - if got := indexResponse.Header().Get("Content-Type"); got != "application/x-yaml" { - t.Errorf("index Content-Type = %q, want application/x-yaml", got) - } - if strings.Contains(indexResponse.Body.String(), upstream.URL) { - t.Errorf("rewritten index contains upstream URL: %s", indexResponse.Body.String()) - } - if !strings.Contains(indexResponse.Body.String(), "example.com/retained") { - t.Errorf("rewritten index lost an unrelated field: %s", indexResponse.Body.String()) - } - - var index map[string]any - if err := yaml.Unmarshal(indexResponse.Body.Bytes(), &index); err != nil { - t.Fatalf("parse rewritten index: %v", err) - } - entries := index["entries"].(map[string]any) - release := entries["demo"].([]any)[0].(map[string]any) - urls := release["urls"].([]any) - wantURL := "http://proxy.example/helm/stable/charts/" + digest + "/demo-1.0.0.tgz" - for _, rawURL := range urls { - if rawURL != wantURL { - t.Errorf("rewritten URL = %q, want %q", rawURL, wantURL) - } - } - - firstChart := serveHelmRequest(h, "/stable/charts/"+digest+"/demo-1.0.0.tgz") - if firstChart.Code != http.StatusOK { - t.Fatalf("chart status = %d, want 200: %s", firstChart.Code, firstChart.Body.String()) - } - if got := firstChart.Body.String(); got != string(chart) { - t.Errorf("chart body = %q, want %q", got, chart) - } - if got := firstChart.Header().Get("Content-Type"); got != "application/gzip" { - t.Errorf("chart Content-Type = %q, want application/gzip", got) - } - - // Artifact cache availability must not depend on metadata caching or a - // reachable index upstream. - proxy.CacheMetadata = false - available.Store(false) - cachedChart := serveHelmRequest(h, "/stable/charts/"+digest+"/demo-1.0.0.tgz") - if cachedChart.Code != http.StatusOK { - t.Fatalf("cached chart status = %d, want 200: %s", cachedChart.Code, cachedChart.Body.String()) - } - if got := cachedChart.Body.String(); got != string(chart) { - t.Errorf("cached chart body = %q, want %q", got, chart) - } - if got := indexRequests.Load(); got != 1 { - t.Errorf("index requests = %d, want 1", got) - } - if got := chartRequests.Load(); got != 1 { - t.Errorf("chart requests = %d, want 1", got) - } -} - -func TestHelmHandler_RejectsChartDigestMismatch(t *testing.T) { - chart := []byte("tampered chart") - digest := helmSHA256Hex([]byte("expected chart")) - requests := 0 - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/index.yaml": - _, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", digest) - case "/demo.tgz": - requests++ - _, _ = w.Write(chart) - default: - http.NotFound(w, r) - } - })) - defer upstream.Close() - - proxy, _, store, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"test": upstream.URL}) - - for range 2 { - response := serveHelmRequest(h, "/test/charts/"+digest+"/demo.tgz") - if response.Code != http.StatusBadGateway { - t.Errorf("status = %d, want 502: %s", response.Code, response.Body.String()) - } - } - if requests != 2 { - t.Errorf("chart requests = %d, want 2 after invalid cache entry is cleared", requests) - } - queued, err := proxy.DB.GetDuePendingDeletes(time.Now().Add(time.Hour), 10) - if err != nil { - t.Fatalf("listing queued deletes: %v", err) - } - chartDir := helmMetadataEcosystem + "/test/" + digest + "/" - stored := 0 - for path := range store.files { - if !strings.HasPrefix(path, chartDir) { - continue - } - stored++ - if !slices.Contains(queued, path) { - t.Errorf("rejected chart at %q is not queued for deletion", path) - } - } - if stored != 0 { - t.Errorf("found %d stored charts, want none: a mismatch is discarded before it is committed", stored) - } -} - -func TestHelmHandler_IndexCacheChangesWithUpstreamURL(t *testing.T) { - firstDigest := strings.Repeat("a", sha256HexLength) - secondDigest := strings.Repeat("b", sha256HexLength) - firstRequests := 0 - secondRequests := 0 - first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - firstRequests++ - _, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", firstDigest) - })) - defer first.Close() - second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - secondRequests++ - _, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", secondDigest) - })) - defer second.Close() - - proxy, db, store, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = first.Client() - firstHandler := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": first.URL}) - if response := serveHelmRequest(firstHandler, "/stable/index.yaml"); response.Code != http.StatusOK { - t.Fatalf("first index status = %d, want 200: %s", response.Code, response.Body.String()) - } - - // Model a restarted server with the same database and storage but a changed - // repository URL. Its cache key must not reuse the previous index or ETag. - restartedProxy := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), nil) - restartedProxy.CacheMetadata = true - restartedProxy.MetadataTTL = time.Hour - restartedProxy.HTTPClient = second.Client() - secondHandler := NewHelmHandler(restartedProxy, "http://proxy.example", map[string]string{"stable": second.URL}) - response := serveHelmRequest(secondHandler, "/stable/index.yaml") - if response.Code != http.StatusOK { - t.Fatalf("second index status = %d, want 200: %s", response.Code, response.Body.String()) - } - if !strings.Contains(response.Body.String(), secondDigest) { - t.Errorf("second index did not use the new upstream: %s", response.Body.String()) - } - if firstRequests != 1 { - t.Errorf("first upstream requests = %d, want 1", firstRequests) - } - if secondRequests != 1 { - t.Errorf("second upstream requests = %d, want 1", secondRequests) - } -} - -func TestHelmHandler_UsesConfiguredUpstreamAuthentication(t *testing.T) { - chart := []byte("private Helm chart") - digest := helmSHA256Hex(chart) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Authorization") != "Bearer private-token" { - http.Error(w, "unauthorized", http.StatusUnauthorized) - return - } - switch r.URL.Path { - case "/index.yaml": - _, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", digest) - case "/demo.tgz": - _, _ = w.Write(chart) - default: - http.NotFound(w, r) - } - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - authClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, - upstreamhttp.AuthFunc(func(string) (string, string) { - return "Authorization", "Bearer private-token" - }))} - proxy.HTTPClient = authClient - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(authClient), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"private": upstream.URL}) - - response := serveHelmRequest(h, "/private/charts/"+digest+"/demo.tgz") - if response.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", response.Code, response.Body.String()) - } - if got := response.Body.String(); got != string(chart) { - t.Errorf("body = %q, want %q", got, chart) - } -} - -func TestHelmHandler_FiltersNewChartsFromIndex(t *testing.T) { - oldDigest := strings.Repeat("a", 64) - newDigest := strings.Repeat("b", 64) - proxy := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}} - h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"test": "https://charts.example"}) - - body := fmt.Sprintf(`apiVersion: v1 -entries: - demo: - - created: %s - digest: %s - urls: [demo-old.tgz] - - created: %s - digest: %s - urls: [demo-new.tgz] -`, time.Now().Add(-10*24*time.Hour).Format(time.RFC3339), oldDigest, - time.Now().Add(-time.Hour).Format(time.RFC3339), newDigest) - - rewritten, _, err := h.rewriteIndex("test", "https://charts.example", []byte(body)) - if err != nil { - t.Fatalf("rewriteIndex() error = %v", err) - } - if strings.Contains(string(rewritten), newDigest) { - t.Errorf("rewritten index includes a chart still in cooldown: %s", rewritten) - } - if !strings.Contains(string(rewritten), oldDigest) { - t.Errorf("rewritten index omitted an old chart: %s", rewritten) - } -} - -func TestNormalizeHelmDigest(t *testing.T) { - digest := strings.Repeat("a", 64) - for _, input := range []string{digest, "sha256:" + digest, "SHA256:" + strings.ToUpper(digest)} { - if got, ok := normalizeHelmDigest(input); !ok || got != digest { - t.Errorf("normalizeHelmDigest(%q) = %q, %t; want %q, true", input, got, ok, digest) - } - } - if _, ok := normalizeHelmDigest("bad"); ok { - t.Error("normalizeHelmDigest accepted an invalid digest") - } -} - -func TestHelmIndexEntriesRejectsIncompleteMapping(t *testing.T) { - entries := &yaml.Node{ - Kind: yaml.MappingNode, - Content: []*yaml.Node{ - {Kind: yaml.ScalarNode, Value: "demo"}, - }, - } - document := &yaml.Node{ - Kind: yaml.DocumentNode, - Content: []*yaml.Node{{ - Kind: yaml.MappingNode, - Content: []*yaml.Node{ - {Kind: yaml.ScalarNode, Value: "entries"}, - entries, - }, - }}, - } - - if _, err := helmIndexEntries(document); err == nil { - t.Fatal("helmIndexEntries() error = nil, want incomplete mapping error") - } -} - -func serveHelmRequest(h *HelmHandler, target string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - return w -} - -func helmSHA256Hex(data []byte) string { - digest := sha256.Sum256(data) - return hex.EncodeToString(digest[:]) -} diff --git a/internal/handler/hex.go b/internal/handler/hex.go index d394c4d..0f0c72e 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -5,11 +5,12 @@ import ( "compress/gzip" "encoding/json" "fmt" + "io" "net/http" "strings" "time" - "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" "google.golang.org/protobuf/encoding/protowire" ) @@ -17,18 +18,10 @@ const ( hexUpstream = "https://repo.hex.pm" ) -// Tarballs are stored under pkg:hex/{name} with the name from the tarball -// filename, and purl keeps hex names as they are, so package rules use the -// default key mapping. -func init() { - retention.Default.Register(retention.Spec{Key: "hex", CanonicalPackage: retention.DefaultCanonical("hex")}) -} - // HexHandler handles Hex.pm registry protocol requests. type HexHandler struct { proxy *Proxy upstreamURL string - apiURL string proxyURL string } @@ -37,20 +30,10 @@ func NewHexHandler(proxy *Proxy, proxyURL string) *HexHandler { return &HexHandler{ proxy: proxy, upstreamURL: hexUpstream, - apiURL: hexAPIURL, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } -// NewHexHandlerWithUpstreams creates a Hex handler with custom repository and -// API upstreams. -func NewHexHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, apiURL string) *HexHandler { - h := NewHexHandler(proxy, proxyURL) - h.upstreamURL = configuredUpstreamURL(upstreamURL, hexUpstream) - h.apiURL = configuredUpstreamURL(apiURL, hexAPIURL) - return h -} - // Routes returns the HTTP handler for Hex requests. func (h *HexHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -71,29 +54,44 @@ func (h *HexHandler) Routes() http.Handler { // handleDownload serves a package tarball, fetching and caching from upstream if needed. func (h *HexHandler) handleDownload(w http.ResponseWriter, r *http.Request) { - h.proxy.handleFilenameDownload(w, r, filenameDownload{ - ecosystem: "hex", - upstreamURL: h.upstreamURL, - suffix: ".tar", - parseErr: "could not parse tarball filename", - fetchErr: "failed to fetch package", - parse: h.parseTarballFilename, - publishTime: h.publishTime, - }) + filename := r.PathValue("filename") + if filename == "" || !strings.HasSuffix(filename, ".tar") { + http.Error(w, "invalid filename", http.StatusBadRequest) + return + } + + // Extract name and version from filename (e.g., "phoenix-1.7.10.tar") + name, version := h.parseTarballFilename(filename) + if name == "" || version == "" { + http.Error(w, "could not parse tarball filename", http.StatusBadRequest) + return + } + + h.proxy.Logger.Info("hex download request", + "name", name, "version", version, "filename", filename) + + result, err := h.proxy.GetOrFetchArtifact(r.Context(), "hex", name, version, filename) + if err != nil { + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch package", http.StatusBadGateway) + return + } + + ServeArtifact(w, result) } // parseTarballFilename extracts name and version from a hex tarball filename. // e.g., "phoenix-1.7.10.tar" -> ("phoenix", "1.7.10") -// e.g., "amqp_client-4.1.5-1.tar" -> ("amqp_client", "4.1.5-1") -// -// Hex package names can't contain a hyphen, so the first one separates the -// name from a version that may carry hyphens of its own. func (h *HexHandler) parseTarballFilename(filename string) (name, version string) { - name, version, ok := strings.Cut(strings.TrimSuffix(filename, ".tar"), "-") - if !ok || name == "" || version == "" { - return "", "" + base := strings.TrimSuffix(filename, ".tar") + + // Find the last hyphen followed by a version number + for i := len(base) - 1; i >= 0; i-- { + if base[i] == '-' && i+1 < len(base) && base[i+1] >= '0' && base[i+1] <= '9' { + return base[:i], base[i+1:] + } } - return name, version + return "", "" } // hexAPIURL is the Hex HTTP API base URL for fetching package metadata with timestamps. @@ -102,116 +100,105 @@ const hexAPIURL = "https://hex.pm" // handlePackages proxies the /packages/{name} endpoint, applying cooldown filtering // when enabled. Since the protobuf format has no timestamps, we fetch them from the // Hex HTTP API concurrently. -// -// Both are read through the metadata cache, so within metadata_ttl a package -// costs no request to repo.hex.pm or to the rate limited Hex API, and after -// it both are revalidated with their ETags. The filtered package is kept in -// the rewrite cache and served with an ETag of its own. func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + h.proxyCached(w, r) + return + } + name := r.PathValue("name") - if name == "" || !h.proxy.cooldownApplies("hex", name) { + if name == "" { h.proxyCached(w, r) return } h.proxy.Logger.Info("hex package request with cooldown", "name", name) - docs, err := h.proxy.fetchTimedMetadata(r.Context(), h.timedPackage(name), h.packageVersions) - switch { - case docs.doc.err != nil: - h.proxy.writeMetadataFetchError(w, docs.doc.err) - return - case docs.times.err != nil: - h.proxy.writeCooldownLookupError(w, "hex", name, docs.times.err) - return - case err != nil: - h.proxy.Logger.Error("failed to read hex package", "name", name, "error", err) - http.Error(w, "failed to filter package metadata", http.StatusBadGateway) - return - } - - packagePURL := canonicalPackagePURL("hex", name) - m, err := h.proxy.cachedRewrite(r.Context(), "hex", h.proxyURL, "packages/"+name, docs.doc.body, docs.rewriteDigest(), - func([]byte) ([]byte, []string, error) { - body, err := h.proxy.decodedMetadataBody(docs.doc) - if err != nil { - return nil, nil, err - } - versions, err := h.packageVersions(docs.doc) - if err != nil { - return nil, nil, err - } - filtered := make(map[string]bool) - var kept []string - for _, version := range versions { - if docs.withheld(h.proxy, "hex", packagePURL, version) { - filtered[version] = true - h.proxy.Logger.Info("cooldown: filtering hex version", "package", name, "version", version) - continue - } - kept = append(kept, version) - } - if len(filtered) == 0 { - // Nothing in cooldown: upstream's bytes, signature included. - return body, kept, nil - } - out, err := h.filterSignedPackage(body, filtered) - return out, kept, err - }) + protoResp, filteredVersions, err := h.fetchPackageAndVersions(r, name) if err != nil { - h.proxy.Logger.Error("failed to filter hex package", "name", name, "error", err) - http.Error(w, "failed to filter package metadata", http.StatusBadGateway) + h.proxy.Logger.Error("upstream request failed", "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + defer func() { _ = protoResp.Body.Close() }() + + if protoResp.StatusCode != http.StatusOK { + for k, vv := range protoResp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + w.WriteHeader(protoResp.StatusCode) + _, _ = io.Copy(w, protoResp.Body) return } - markStale(w, docs.doc.stale || docs.times.stale) - serveHexPackage(w, r, m) -} - -// serveHexPackage writes a package with its validators, or 304 when they -// show the client's copy is current. The registry file is gzip itself, and -// Hex clients gunzip it however it is labelled, so it is never compressed -// again. -func serveHexPackage(w http.ResponseWriter, r *http.Request, m rewrittenMetadata) { - w.Header().Set(headerETag, m.etag) - if !m.lastModified.IsZero() { - w.Header().Set(headerLastModified, m.lastModified.UTC().Format(http.TimeFormat)) - } - if notModified(r, m.etag, m.lastModified) { - w.WriteHeader(http.StatusNotModified) + body, err := io.ReadAll(protoResp.Body) + if err != nil { + http.Error(w, "failed to read response", http.StatusInternalServerError) return } - w.Header().Set(headerContentType, "application/octet-stream") - w.Header().Set(headerContentEncoding, "gzip") - w.WriteHeader(http.StatusOK) - _, _ = w.Write(m.content()) -} -// timedPackage describes a package's registry file and the Hex API entry -// it takes publish times from. -func (h *HexHandler) timedPackage(name string) timedMetadata { - return timedMetadata{ - ecosystem: "hex", - docKey: "packages/" + name, - docURL: h.upstreamURL + "/packages/" + name, - timesKey: hashedCacheKey(h.apiURL, "api/packages/"+name), - timesURL: fmt.Sprintf("%s/api/packages/%s", h.apiURL, name), - timesAccept: contentTypeJSON, - parseTimes: parseHexReleases, + if len(filteredVersions) == 0 { + // No versions to filter or couldn't get timestamps, pass through + w.Header().Set("Content-Type", protoResp.Header.Get("Content-Type")) + w.Header().Set("Content-Encoding", "gzip") + _, _ = w.Write(body) + return } + + filtered, err := h.filterSignedPackage(body, filteredVersions) + if err != nil { + h.proxy.Logger.Warn("failed to filter hex package, proxying original", "error", err) + w.Header().Set("Content-Type", protoResp.Header.Get("Content-Type")) + w.Header().Set("Content-Encoding", "gzip") + _, _ = w.Write(body) + return + } + + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Encoding", "gzip") + _, _ = w.Write(filtered) } -// packageVersions lists the releases in a gzipped Signed package. -func (h *HexHandler) packageVersions(doc metadataResult) ([]string, error) { - body, err := h.proxy.decodedMetadataBody(doc) +// fetchPackageAndVersions fetches the protobuf package and version timestamps concurrently. +func (h *HexHandler) fetchPackageAndVersions(r *http.Request, name string) (*http.Response, map[string]bool, error) { + type versionsResult struct { + filtered map[string]bool + err error + } + + versionsCh := make(chan versionsResult, 1) + go func() { + filtered, err := h.fetchFilteredVersions(r, name) + versionsCh <- versionsResult{filtered: filtered, err: err} + }() + + protoResp, err := h.fetchUpstreamPackage(r, name) + + versionsRes := <-versionsCh + + if err != nil { + return nil, nil, err + } + + if versionsRes.err != nil { + h.proxy.Logger.Warn("failed to fetch hex version timestamps, proxying unfiltered", + "name", name, "error", versionsRes.err) + return protoResp, nil, nil + } + + return protoResp, versionsRes.filtered, nil +} + +// fetchUpstreamPackage fetches the protobuf package from upstream. +func (h *HexHandler) fetchUpstreamPackage(r *http.Request, name string) (*http.Response, error) { + upstreamURL := h.upstreamURL + "/packages/" + name + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) if err != nil { return nil, err } - payload, err := h.signedPayload(body) - if err != nil { - return nil, err - } - return packageReleaseVersions(payload) + return h.proxy.HTTPClient.Do(req) } // hexRelease represents a version entry from the Hex API. @@ -225,35 +212,61 @@ type hexPackageAPI struct { Releases []hexRelease `json:"releases"` } -// parseHexReleases reads the publish times from a Hex API package response. -func parseHexReleases(body []byte) (publishTimes, error) { - var pkg hexPackageAPI - if err := json.Unmarshal(body, &pkg); err != nil { +// fetchFilteredVersions fetches the Hex API and returns a set of version +// strings that should be filtered out by cooldown. +func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[string]bool, error) { + apiURL := fmt.Sprintf("%s/api/packages/%s", hexAPIURL, name) + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, apiURL, nil) + if err != nil { return nil, err } - times := make(publishTimes, len(pkg.Releases)) + req.Header.Set("Accept", "application/json") + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("hex API returned %d", resp.StatusCode) + } + + var pkg hexPackageAPI + if err := json.NewDecoder(resp.Body).Decode(&pkg); err != nil { + return nil, err + } + + packagePURL := purl.MakePURLString("hex", name, "") + filtered := make(map[string]bool) + for _, release := range pkg.Releases { insertedAt, err := time.Parse(time.RFC3339Nano, release.InsertedAt) if err != nil { - insertedAt = time.Time{} + continue + } + + if !h.proxy.Cooldown.IsAllowed("hex", packagePURL, insertedAt) { + filtered[release.Version] = true + h.proxy.Logger.Info("cooldown: filtering hex version", + "package", name, "version", release.Version, + "published", release.InsertedAt) } - times[release.Version] = insertedAt } - return times, nil + + return filtered, nil } -func (h *HexHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, bool, error) { - return h.proxy.lookupPublishTime(r.Context(), h.timedPackage(name), version, h.packageVersions) -} - -// signedPayload decompresses a gzipped Signed message, with the metadata -// limit applied to the decompressed size, and returns its payload. -func (h *HexHandler) signedPayload(gzippedData []byte) ([]byte, error) { +// filterSignedPackage decompresses gzipped data, decodes the Signed protobuf wrapper, +// filters releases from the Package payload, and re-encodes as gzipped protobuf +// (without the original signature since the payload has changed). +func (h *HexHandler) filterSignedPackage(gzippedData []byte, filteredVersions map[string]bool) ([]byte, error) { + // Decompress gzip gr, err := gzip.NewReader(bytes.NewReader(gzippedData)) if err != nil { return nil, err } - signed, err := h.proxy.ReadMetadata(gr) + signed, err := io.ReadAll(gr) if err != nil { return nil, err } @@ -264,30 +277,9 @@ func (h *HexHandler) signedPayload(gzippedData []byte) ([]byte, error) { if err != nil { return nil, fmt.Errorf("extracting payload: %w", err) } - return payload, nil -} - -// packageReleaseVersions lists the versions of the releases in a Package -// message. -func packageReleaseVersions(payload []byte) ([]string, error) { - var versions []string - _, err := filterPackageReleases(payload, nil, func(version string) { - versions = append(versions, version) - }) - return versions, err -} - -// filterSignedPackage decompresses gzipped data, decodes the Signed protobuf wrapper, -// filters releases from the Package payload, and re-encodes as gzipped protobuf -// (without the original signature since the payload has changed). -func (h *HexHandler) filterSignedPackage(gzippedData []byte, filteredVersions map[string]bool) ([]byte, error) { - payload, err := h.signedPayload(gzippedData) - if err != nil { - return nil, err - } // Filter releases from the Package message - filteredPayload, err := filterPackageReleases(payload, filteredVersions, nil) + filteredPayload, err := filterPackageReleases(payload, filteredVersions) if err != nil { return nil, fmt.Errorf("filtering releases: %w", err) } @@ -310,10 +302,9 @@ func (h *HexHandler) filterSignedPackage(gzippedData []byte, filteredVersions ma return buf.Bytes(), nil } -// filterPackageReleases filters releases from a Package protobuf message, -// calling seen, when not nil, with the version of each release. +// filterPackageReleases filters releases from a Package protobuf message. // Package: field 1 = releases (repeated), field 2 = name, field 3 = repository -func filterPackageReleases(payload []byte, filteredVersions map[string]bool, seen func(string)) ([]byte, error) { +func filterPackageReleases(payload []byte, filteredVersions map[string]bool) ([]byte, error) { var result []byte data := payload @@ -338,9 +329,6 @@ func filterPackageReleases(payload []byte, filteredVersions map[string]bool, see if num == 1 { // releases field version := extractReleaseVersion(v) - if seen != nil { - seen(version) - } if filteredVersions[version] { continue // skip this release } diff --git a/internal/handler/hex_test.go b/internal/handler/hex_test.go index a0b0448..b02540a 100644 --- a/internal/handler/hex_test.go +++ b/internal/handler/hex_test.go @@ -3,7 +3,6 @@ package handler import ( "bytes" "compress/gzip" - cryptorand "crypto/rand" "encoding/json" "io" "log/slog" @@ -27,11 +26,6 @@ func TestHexParseTarballFilename(t *testing.T) { {"phoenix-1.7.10.tar", "phoenix", "1.7.10"}, {"ecto-3.11.0.tar", "ecto", "3.11.0"}, {"phoenix_live_view-0.20.1.tar", "phoenix_live_view", "0.20.1"}, - {"amqp_client-4.1.5-1.tar", "amqp_client", "4.1.5-1"}, - {"bandit-0.1.0-dev-10.tar", "bandit", "0.1.0-dev-10"}, - {"ex_doc-0.34.0-rc.1.tar", "ex_doc", "0.34.0-rc.1"}, - {"-1.0.0.tar", "", ""}, - {"phoenix-.tar", "", ""}, {"invalid", "", ""}, } @@ -95,7 +89,7 @@ func buildHexSigned(payload []byte) []byte { func TestHexFilterPackageReleases(t *testing.T) { pkg := buildHexPackage("phoenix", []string{testVersion100, "2.0.0", "3.0.0"}) - filtered, err := filterPackageReleases(pkg, map[string]bool{"2.0.0": true}, nil) + filtered, err := filterPackageReleases(pkg, map[string]bool{"2.0.0": true}) if err != nil { t.Fatal(err) } @@ -330,67 +324,3 @@ func TestHexHandlePackagesWithoutCooldown(t *testing.T) { t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } } - -func TestHexCooldownMetadataSizeLimit(t *testing.T) { - const limit = 1024 - now := time.Now() - releases := []hexRelease{ - {Version: testVersion100, InsertedAt: now.Add(-7 * 24 * time.Hour).Format(time.RFC3339Nano)}, - {Version: "2.0.0", InsertedAt: now.Add(-time.Hour).Format(time.RFC3339Nano)}, - } - apiJSON, _ := json.Marshal(hexPackageAPI{Releases: releases}) - pkg := buildHexPackage("testpkg", []string{testVersion100, "2.0.0"}) - proto := buildHexSigned(pkg) - - // An unknown bytes field pads the payload: it compresses to a few bytes - // but decompresses past the limit. - padded := protowire.AppendTag(append([]byte(nil), pkg...), 4, protowire.BytesType) - padded = protowire.AppendBytes(padded, make([]byte, 64*limit)) - bomb := buildHexSigned(padded) - if len(bomb) >= limit { - t.Fatalf("compressed package is %d bytes, want under %d", len(bomb), limit) - } - // Padding that doesn't compress makes the response itself too large. - noise := make([]byte, 2*limit) - _, _ = cryptorand.Read(noise) - large := protowire.AppendTag(append([]byte(nil), pkg...), 4, protowire.BytesType) - large = buildHexSigned(protowire.AppendBytes(large, noise)) - if len(large) <= limit { - t.Fatalf("compressed package is %d bytes, want over %d", len(large), limit) - } - - tests := []struct { - name string - pkg, api []byte - }{ - {"package over limit", large, apiJSON}, - {"package decompresses past limit", bomb, apiJSON}, - {"API response over limit", proto, append(append([]byte(nil), apiJSON...), bytes.Repeat([]byte(" "), 2*limit)...)}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/packages/testpkg": - _, _ = w.Write(tt.pkg) - case "/api/packages/testpkg": - _, _ = w.Write(tt.api) - default: - w.WriteHeader(http.StatusNotFound) - } - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - proxy.MetadataMaxSize = limit - h := NewHexHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) - - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/packages/testpkg", nil)) - if w.Code != http.StatusBadGateway { - t.Errorf("status = %d, want %d", w.Code, http.StatusBadGateway) - } - }) - } -} diff --git a/internal/handler/homebrew.go b/internal/handler/homebrew.go deleted file mode 100644 index 4e40d02..0000000 --- a/internal/handler/homebrew.go +++ /dev/null @@ -1,74 +0,0 @@ -package handler - -import ( - "crypto/sha256" - "encoding/hex" - "net/http" - "strings" -) - -const ( - homebrewArtifactNamespace = "homebrew" - homebrewArtifactRepository = "homebrew/core" - homebrewMetadataEcosystem = "homebrew" -) - -// HomebrewHandler proxies Homebrew's JSON API without modifying signed files. -type HomebrewHandler struct { - proxy *Proxy - apiUpstream string -} - -// NewHomebrewHandler creates a Homebrew JSON API handler. -func NewHomebrewHandler(proxy *Proxy, apiUpstream string) *HomebrewHandler { - return &HomebrewHandler{ - proxy: proxy, - apiUpstream: strings.TrimSuffix(apiUpstream, "/"), - } -} - -// RegisterHomebrewArtifacts routes homebrew/core OCI requests to its configured -// registry and blocks other homebrew repositories from reaching the default -// OCI registry. -func RegisterHomebrewArtifacts(container *ContainerHandler, artifactUpstream string) { - container.BlockRegistry(homebrewArtifactNamespace) - container.RegisterRegistry(homebrewArtifactRepository, artifactUpstream) -} - -// Routes returns the Homebrew JSON API handler. Mount this at /homebrew. -func (h *HomebrewHandler) Routes() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet && r.Method != http.MethodHead { - w.Header().Set("Allow", "GET, HEAD") - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - - requestPath := strings.TrimPrefix(r.URL.EscapedPath(), "/") - if requestPath == "" || containsPathTraversal(requestPath) { - http.NotFound(w, r) - return - } - - upstreamURL := h.apiUpstream + "/" + requestPath - if r.URL.RawQuery != "" { - upstreamURL += "?" + r.URL.RawQuery - } - - // brew fetches every JSON API download with `curl --compressed` and - // decodes Content-Encoding itself, and formula.jws.json is ~33 MB plain - // versus ~5 MB gzip, so keep both hops compressed. The analytics - // endpoints are the one consumer brew fetches without --compressed; - // they stay identity. - acceptEncoding := "gzip" - if strings.HasPrefix(requestPath, "analytics/") { - acceptEncoding = "identity" - } - h.proxy.proxyCachedWithEncoding(w, r, upstreamURL, homebrewMetadataEcosystem, homebrewMetadataCacheKey(requestPath, r.URL.RawQuery), acceptEncoding, nil, "*/*") - }) -} - -func homebrewMetadataCacheKey(requestPath, rawQuery string) string { - sum := sha256.Sum256([]byte(requestPath + "\x00" + rawQuery)) - return hex.EncodeToString(sum[:]) -} diff --git a/internal/handler/homebrew_bottles.go b/internal/handler/homebrew_bottles.go deleted file mode 100644 index 169ed61..0000000 --- a/internal/handler/homebrew_bottles.go +++ /dev/null @@ -1,92 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "regexp" - "strconv" - "strings" -) - -var ( - homebrewFormulaPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9@+._-]*$`) - homebrewVersionPattern = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z._+-]*$`) -) - -// homebrewMaxFormulaJSON bounds one formula's JSON API document, a few KB. -const homebrewMaxFormulaJSON = 1 << 20 - -// homebrewFormula is the part of a formula's JSON API document that names -// its bottles' image tag. -type homebrewFormula struct { - Versions struct { - Stable string `json:"stable"` - } `json:"versions"` - Revision int `json:"revision"` - Bottle struct { - Stable struct { - Rebuild int `json:"rebuild"` - Files map[string]any `json:"files"` - } `json:"stable"` - } `json:"bottle"` -} - -// BottleImage returns the homebrew/core image and tag for a formula version. -// Only the current version gets its revision and rebuild from the JSON API; -// any other version is used as the tag as given. -func (h *HomebrewHandler) BottleImage(ctx context.Context, formula, version string) (ImageRef, error) { - if !homebrewFormulaPattern.MatchString(formula) { - return ImageRef{}, fmt.Errorf("invalid Homebrew formula name %q", formula) - } - if !homebrewVersionPattern.MatchString(version) { - return ImageRef{}, fmt.Errorf("invalid Homebrew version %q", version) - } - f, err := h.fetchFormula(ctx, formula) - if err != nil { - return ImageRef{}, err - } - tag := version - if version == f.Versions.Stable { - if len(f.Bottle.Stable.Files) == 0 { - return ImageRef{}, fmt.Errorf("homebrew formula %s %s has no bottles", formula, version) - } - if f.Revision > 0 { - tag += "_" + strconv.Itoa(f.Revision) - } - if f.Bottle.Stable.Rebuild > 0 { - tag += "-" + strconv.Itoa(f.Bottle.Stable.Rebuild) - } - } - name := strings.NewReplacer("@", "/", "+", "x").Replace(formula) - return ImageRef{Name: homebrewArtifactRepository + "/" + name, References: []string{tag}}, nil -} - -func (h *HomebrewHandler) fetchFormula(ctx context.Context, formula string) (*homebrewFormula, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.apiUpstream+"/formula/"+formula+".json", nil) - if err != nil { - return nil, err - } - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return nil, fmt.Errorf("reading the Homebrew formula: %w", err) - } - defer func() { _ = resp.Body.Close() }() - switch { - case resp.StatusCode == http.StatusNotFound: - return nil, fmt.Errorf("homebrew/core has no formula %q", formula) - case resp.StatusCode != http.StatusOK: - return nil, fmt.Errorf("reading the Homebrew formula: upstream answered %d", resp.StatusCode) - } - var f homebrewFormula - if err := json.NewDecoder(io.LimitReader(resp.Body, homebrewMaxFormulaJSON)).Decode(&f); err != nil { - return nil, fmt.Errorf("reading the Homebrew formula: %w", err) - } - if f.Versions.Stable == "" { - return nil, errors.New("reading the Homebrew formula: it has no stable version") - } - return &f, nil -} diff --git a/internal/handler/homebrew_bottles_test.go b/internal/handler/homebrew_bottles_test.go deleted file mode 100644 index a2ddb64..0000000 --- a/internal/handler/homebrew_bottles_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package handler - -import ( - "context" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" -) - -func TestHomebrewBottleImage(t *testing.T) { - formulas := map[string]string{ - "/formula/hello.json": `{"versions":{"stable":"2.12.3"},"revision":0, - "bottle":{"stable":{"rebuild":1,"files":{"arm64_linux":{}}}}}`, - "/formula/openssl@3.json": `{"versions":{"stable":"3.5.0"},"revision":2, - "bottle":{"stable":{"rebuild":0,"files":{"arm64_sequoia":{}}}}}`, - "/formula/libsigc++.json": `{"versions":{"stable":"3.6.0"},"revision":0, - "bottle":{"stable":{"rebuild":0,"files":{"x86_64_linux":{}}}}}`, - "/formula/nobottle.json": `{"versions":{"stable":"1.0"},"bottle":{}}`, - } - var requests atomic.Int32 - api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - body, ok := formulas[r.URL.Path] - if !ok { - http.NotFound(w, r) - return - } - _, _ = w.Write([]byte(body)) - })) - defer api.Close() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = api.Client() - h := NewHomebrewHandler(proxy, api.URL) - - tests := []struct { - formula, version string - image, tag, err string - }{ - {formula: "hello", version: "2.12.3", image: "homebrew/core/hello", tag: "2.12.3-1"}, - {formula: "openssl@3", version: "3.5.0", image: "homebrew/core/openssl/3", tag: "3.5.0_2"}, - {formula: "libsigc++", version: "3.6.0", image: "homebrew/core/libsigcxx", tag: "3.6.0"}, - // Only the current version is described; another is the tag as given. - {formula: "hello", version: "2.12.2", image: "homebrew/core/hello", tag: "2.12.2"}, - {formula: "hello", version: "2.12.3-1", image: "homebrew/core/hello", tag: "2.12.3-1"}, - {formula: "nobottle", version: "1.0", err: "has no bottles"}, - {formula: "missing", version: "1.0", err: `no formula "missing"`}, - } - for _, tt := range tests { - ref, err := h.BottleImage(context.Background(), tt.formula, tt.version) - if tt.err != "" { - if err == nil || !strings.Contains(err.Error(), tt.err) { - t.Errorf("%s@%s: err = %v, want %q", tt.formula, tt.version, err, tt.err) - } - continue - } - if err != nil || ref.Name != tt.image || len(ref.References) != 1 || ref.References[0] != tt.tag { - t.Errorf("%s@%s = %+v, %v; want %s:%s", tt.formula, tt.version, ref, err, tt.image, tt.tag) - } - } - - before := requests.Load() - for _, bad := range [][2]string{{"../hello", "1.0"}, {"Hello", "1.0"}, {"hello", "1.0/../x"}, {"hello", ""}} { - if _, err := h.BottleImage(context.Background(), bad[0], bad[1]); err == nil { - t.Errorf("%q@%q accepted", bad[0], bad[1]) - } - } - if requests.Load() != before { - t.Error("an invalid name or version reached the API") - } -} diff --git a/internal/handler/homebrew_test.go b/internal/handler/homebrew_test.go deleted file mode 100644 index e5d7e5c..0000000 --- a/internal/handler/homebrew_test.go +++ /dev/null @@ -1,458 +0,0 @@ -package handler - -import ( - "bytes" - "io" - "net/http" - "net/http/httptest" - "strconv" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/registries/fetch" -) - -func TestHomebrewHandler_PreservesSignedResponseAndClientValidators(t *testing.T) { - body := " {\n \"payload\": \"signed bytes\",\n \"signatures\": []\n}\n" - etag := `"homebrew-api-etag"` - lastModified := time.Date(2026, time.August, 14, 9, 30, 0, 0, time.UTC) - requests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests++ - if r.Method != http.MethodGet { - t.Errorf("upstream method = %s, want GET", r.Method) - } - if r.URL.Path != "/api/internal/packages.arm64_tahoe.jws.json" { - t.Errorf("upstream path = %q", r.URL.Path) - } - if got := r.Header.Get("Authorization"); got != "" { - t.Errorf("upstream Authorization = %q, want empty", got) - } - if got := r.Header.Get("Cookie"); got != "" { - t.Errorf("upstream Cookie = %q, want empty", got) - } - w.Header().Set("Content-Type", "application/json") - w.Header().Set("ETag", etag) - w.Header().Set("Last-Modified", lastModified.Format(http.TimeFormat)) - _, _ = io.WriteString(w, body) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - h := NewHomebrewHandler(proxy, upstream.URL+"/api").Routes() - - req := httptest.NewRequest(http.MethodGet, "/internal/packages.arm64_tahoe.jws.json", nil) - req.Header.Set("Authorization", "Bearer client-secret") - req.Header.Set("Cookie", "session=client-secret") - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if got := w.Body.String(); got != body { - t.Errorf("body = %q, want byte-for-byte %q", got, body) - } - if got := w.Header().Get("Content-Type"); got != "application/json" { - t.Errorf("Content-Type = %q, want application/json", got) - } - wantContentLength := strconv.Itoa(len(body)) - if got := w.Header().Get("Content-Length"); got != wantContentLength { - t.Errorf("Content-Length = %q, want %q", got, wantContentLength) - } - if got := w.Header().Get("ETag"); got != etag { - t.Errorf("ETag = %q, want %q", got, etag) - } - if got := w.Header().Get("Last-Modified"); got != lastModified.Format(http.TimeFormat) { - t.Errorf("Last-Modified = %q, want %q", got, lastModified.Format(http.TimeFormat)) - } - - conditionalRequest := httptest.NewRequest(http.MethodGet, "/internal/packages.arm64_tahoe.jws.json", nil) - conditionalRequest.Header.Set("If-None-Match", etag) - conditional := httptest.NewRecorder() - h.ServeHTTP(conditional, conditionalRequest) - if conditional.Code != http.StatusNotModified { - t.Fatalf("conditional status = %d, want %d", conditional.Code, http.StatusNotModified) - } - if got := conditional.Header().Get("ETag"); got != etag { - t.Errorf("conditional ETag = %q, want %q", got, etag) - } - if conditional.Body.Len() != 0 { - t.Errorf("conditional body length = %d, want 0", conditional.Body.Len()) - } - - modifiedSinceRequest := httptest.NewRequest(http.MethodGet, "/internal/packages.arm64_tahoe.jws.json", nil) - modifiedSinceRequest.Header.Set("If-Modified-Since", lastModified.Format(http.TimeFormat)) - modifiedSince := httptest.NewRecorder() - h.ServeHTTP(modifiedSince, modifiedSinceRequest) - if modifiedSince.Code != http.StatusNotModified { - t.Fatalf("If-Modified-Since status = %d, want %d", modifiedSince.Code, http.StatusNotModified) - } - if got := modifiedSince.Header().Get("Last-Modified"); got != lastModified.Format(http.TimeFormat) { - t.Errorf("conditional Last-Modified = %q, want %q", got, lastModified.Format(http.TimeFormat)) - } - if requests != 1 { - t.Errorf("upstream requests = %d, want 1", requests) - } -} - -func TestHomebrewHandler_HeadUsesMetadataCacheAndSurvivesOutage(t *testing.T) { - body := `{"payload":"signed bytes","signatures":[]}` - available := true - requests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests++ - if got := r.Header.Get("Authorization"); got != "" { - t.Errorf("upstream Authorization = %q, want empty", got) - } - if !available { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", "application/json") - w.Header().Set("ETag", `"head-etag"`) - _, _ = io.WriteString(w, body) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - h := NewHomebrewHandler(proxy, upstream.URL+"/api").Routes() - - coldRequest := httptest.NewRequest(http.MethodHead, "/formula.jws.json", nil) - coldRequest.Header.Set("Authorization", "Bearer client-secret") - cold := httptest.NewRecorder() - h.ServeHTTP(cold, coldRequest) - if cold.Code != http.StatusOK { - t.Fatalf("cold status = %d, want %d", cold.Code, http.StatusOK) - } - if cold.Body.Len() != 0 { - t.Errorf("cold body length = %d, want 0", cold.Body.Len()) - } - if got := cold.Header().Get("Content-Length"); got != strconv.Itoa(len(body)) { - t.Errorf("Content-Length = %q, want %d", got, len(body)) - } - if got := cold.Header().Get("ETag"); got != `"head-etag"` { - t.Errorf("ETag = %q, want %q", got, `"head-etag"`) - } - if requests != 1 { - t.Fatalf("cold upstream requests = %d, want 1", requests) - } - - warm := httptest.NewRecorder() - h.ServeHTTP(warm, httptest.NewRequest(http.MethodHead, "/formula.jws.json", nil)) - if warm.Code != http.StatusOK { - t.Fatalf("warm status = %d, want %d", warm.Code, http.StatusOK) - } - if warm.Body.Len() != 0 { - t.Errorf("warm body length = %d, want 0", warm.Body.Len()) - } - if requests != 1 { - t.Errorf("warm upstream requests = %d, want 1", requests) - } - - proxy.MetadataTTL = time.Nanosecond - available = false - stale := httptest.NewRecorder() - h.ServeHTTP(stale, httptest.NewRequest(http.MethodHead, "/formula.jws.json", nil)) - if stale.Code != http.StatusOK { - t.Fatalf("stale status = %d, want %d; body: %s", stale.Code, http.StatusOK, stale.Body.String()) - } - if stale.Body.Len() != 0 { - t.Errorf("stale body length = %d, want 0", stale.Body.Len()) - } - if got := stale.Header().Get("Warning"); got != containerStaleWarning { - t.Errorf("Warning = %q, want %q", got, containerStaleWarning) - } -} - -func TestHomebrewHandler_HeadWithoutMetadataCachePreservesUpstreamMethod(t *testing.T) { - upstreamMethod := "" - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamMethod = r.Method - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Content-Length", "42") - w.Header().Set("ETag", `"head-etag"`) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = false - proxy.HTTPClient = upstream.Client() - h := NewHomebrewHandler(proxy, upstream.URL+"/api").Routes() - - head := httptest.NewRecorder() - h.ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/formula.jws.json", nil)) - if head.Code != http.StatusOK { - t.Fatalf("status = %d, want %d", head.Code, http.StatusOK) - } - if upstreamMethod != http.MethodHead { - t.Errorf("upstream method = %q, want HEAD", upstreamMethod) - } - if head.Body.Len() != 0 { - t.Errorf("body length = %d, want 0", head.Body.Len()) - } - if got := head.Header().Get("Content-Length"); got != "42" { - t.Errorf("Content-Length = %q, want 42", got) - } -} - -func TestHomebrewHandler_ServesStaleCachedResponseWhenUpstreamFails(t *testing.T) { - body := `{"payload":"signed bytes","signatures":[]}` - available := true - requests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - requests++ - if !available { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", "application/json") - w.Header().Set("ETag", `"stale-etag"`) - _, _ = io.WriteString(w, body) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = 5 * time.Millisecond - proxy.HTTPClient = upstream.Client() - h := NewHomebrewHandler(proxy, upstream.URL+"/api").Routes() - - first := httptest.NewRecorder() - h.ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/formula.jws.json", nil)) - if first.Code != http.StatusOK { - t.Fatalf("warm status = %d, want %d", first.Code, http.StatusOK) - } - - time.Sleep(10 * time.Millisecond) - available = false - stale := httptest.NewRecorder() - h.ServeHTTP(stale, httptest.NewRequest(http.MethodGet, "/formula.jws.json", nil)) - if stale.Code != http.StatusOK { - t.Fatalf("stale status = %d, want %d; body: %s", stale.Code, http.StatusOK, stale.Body.String()) - } - if got := stale.Body.String(); got != body { - t.Errorf("stale body = %q, want %q", got, body) - } - if got := stale.Header().Get("Warning"); got != containerStaleWarning { - t.Errorf("Warning = %q, want %q", got, containerStaleWarning) - } - if requests != 2 { - t.Errorf("upstream requests = %d, want 2", requests) - } -} - -func TestHomebrewHandler_ProxiesSupportedAPIPaths(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, r.URL.RequestURI()) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := NewHomebrewHandler(proxy, upstream.URL+"/api").Routes() - - paths := []string{ - "/formula.jws.json", - "/cask.jws.json", - "/formula/jq.json", - "/cask/firefox.json", - "/internal/packages.arm64_tahoe.jws.json?download=1", - } - for _, requestPath := range paths { - t.Run(requestPath, func(t *testing.T) { - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, requestPath, nil)) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) - } - if got, want := w.Body.String(), "/api"+requestPath; got != want { - t.Errorf("upstream request = %q, want %q", got, want) - } - }) - } -} - -func TestHomebrewHandler_RejectsUnsupportedRequests(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - h := NewHomebrewHandler(proxy, "https://example.test/api").Routes() - - method := httptest.NewRecorder() - h.ServeHTTP(method, httptest.NewRequest(http.MethodPost, "/formula.jws.json", nil)) - if method.Code != http.StatusMethodNotAllowed { - t.Errorf("POST status = %d, want %d", method.Code, http.StatusMethodNotAllowed) - } - if got := method.Header().Get("Allow"); got != "GET, HEAD" { - t.Errorf("Allow = %q, want GET, HEAD", got) - } - - root := httptest.NewRecorder() - h.ServeHTTP(root, httptest.NewRequest(http.MethodGet, "/", nil)) - if root.Code != http.StatusNotFound { - t.Errorf("root status = %d, want %d", root.Code, http.StatusNotFound) - } - - traversal := httptest.NewRecorder() - h.ServeHTTP(traversal, httptest.NewRequest(http.MethodGet, "/%2e%2e/secret", nil)) - if traversal.Code != http.StatusNotFound { - t.Errorf("traversal status = %d, want %d", traversal.Code, http.StatusNotFound) - } -} - -func TestRegisterHomebrewArtifacts(t *testing.T) { - h := &ContainerHandler{registryURL: dockerHubRegistry} - artifactUpstream := "https://homebrew-proxy.example.com" - RegisterHomebrewArtifacts(h, artifactUpstream+"/") - - if got := h.registryURLFor("homebrew/core/jq"); got != artifactUpstream { - t.Errorf("homebrew/core registry = %q, want %q", got, artifactUpstream) - } - if got := h.registryURLFor("homebrew/cask/firefox"); got != "" { - t.Errorf("other Homebrew registry = %q, want blocked", got) - } - if got := h.registryURLFor("library/nginx"); got != dockerHubRegistry { - t.Errorf("unrelated registry = %q, want %q", got, dockerHubRegistry) - } -} - -func TestRegisterHomebrewArtifactsRejectsOtherHomebrewRoutes(t *testing.T) { - upstreamRequests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - upstreamRequests++ - _, _ = io.WriteString(w, "unexpected upstream response") - })) - defer upstream.Close() - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("unexpected upstream blob")), - ContentType: "application/octet-stream", - } - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL} - RegisterHomebrewArtifacts(h, "https://ghcr.io") - - const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - paths := []string{ - "/homebrew/cask/firefox/blobs/" + digest, - "/homebrew/cask/firefox/manifests/latest", - "/homebrew/cask/firefox/tags/list", - } - for _, path := range paths { - t.Run(path, func(t *testing.T) { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - if w.Code != http.StatusNotFound { - t.Errorf("status = %d, want %d; body: %s", w.Code, http.StatusNotFound, w.Body.String()) - } - }) - } - - if fetcher.fetchCalled { - t.Error("blocked Homebrew blob reached the artifact fetcher") - } - if upstreamRequests != 0 { - t.Errorf("blocked Homebrew routes made %d upstream requests, want 0", upstreamRequests) - } -} - -// TestHomebrewHandler_RequestsGzipForAPIPaths covers #305's motivating case: -// the JSON API files are fetched, cached and served gzip-compressed with -// Content-Encoding: gzip (brew fetches them with --compressed), while the -// analytics endpoints, which brew fetches without --compressed, stay identity. -func TestHomebrewHandler_RequestsGzipForAPIPaths(t *testing.T) { - plain := []byte(`{"payload":"signed bytes","signatures":[]}`) - compressed := gzipPayload(t, plain) - - var available atomic.Bool - available.Store(true) - var requests atomic.Int32 - var sawAcceptEncoding atomic.Value // string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - sawAcceptEncoding.Store(r.Header.Get(headerAcceptEncoding)) - if !available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set(headerContentType, "application/json") - if strings.Contains(r.Header.Get(headerAcceptEncoding), "gzip") { - w.Header().Set(headerContentEncoding, "gzip") - _, _ = w.Write(compressed) - return - } - _, _ = w.Write(plain) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - h := NewHomebrewHandler(proxy, upstream.URL+"/api").Routes() - - get := func(path string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - return w - } - lastAE := func() string { - s, _ := sawAcceptEncoding.Load().(string) - return s - } - - first := get("/formula.jws.json") - if first.Code != http.StatusOK { - t.Fatalf("formula.jws.json: status = %d, want 200: %s", first.Code, first.Body.String()) - } - if got := lastAE(); got != "gzip" { - t.Errorf("formula.jws.json: upstream Accept-Encoding = %q, want %q", got, "gzip") - } - if !bytes.Equal(first.Body.Bytes(), compressed) { - t.Errorf("formula.jws.json: body is not the compressed bytes (got %d, want %d)", first.Body.Len(), len(compressed)) - } - if got := first.Header().Get(headerContentEncoding); got != "gzip" { - t.Errorf("formula.jws.json: Content-Encoding = %q, want %q", got, "gzip") - } - if got := first.Header().Get(headerContentLength); got != strconv.Itoa(len(compressed)) { - t.Errorf("formula.jws.json: Content-Length = %q, want %d", got, len(compressed)) - } - - // Replay from cache with the upstream down: same bytes and header, no refetch. - before := requests.Load() - available.Store(false) - cached := get("/formula.jws.json") - if cached.Code != http.StatusOK { - t.Fatalf("cached formula.jws.json: status = %d, want 200: %s", cached.Code, cached.Body.String()) - } - if !bytes.Equal(cached.Body.Bytes(), compressed) || cached.Header().Get(headerContentEncoding) != "gzip" { - t.Errorf("cached formula.jws.json: body/header not replayed verbatim") - } - if requests.Load() != before { - t.Errorf("cached formula.jws.json hit upstream: requests %d -> %d", before, requests.Load()) - } - available.Store(true) - - // Analytics is fetched by brew without --compressed: stays identity, no header. - analytics := get("/analytics/install/30d.json") - if analytics.Code != http.StatusOK { - t.Fatalf("analytics: status = %d, want 200: %s", analytics.Code, analytics.Body.String()) - } - if got := lastAE(); got != "identity" { - t.Errorf("analytics: upstream Accept-Encoding = %q, want %q", got, "identity") - } - if !bytes.Equal(analytics.Body.Bytes(), plain) { - t.Errorf("analytics: body = %q, want plain %q", analytics.Body.Bytes(), plain) - } - if got := analytics.Header().Get(headerContentEncoding); got != "" { - t.Errorf("analytics: Content-Encoding = %q, want empty", got) - } -} diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go deleted file mode 100644 index 33f2191..0000000 --- a/internal/handler/hosted.go +++ /dev/null @@ -1,499 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha512" - "database/sql" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "slices" - "strings" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/metrics" - "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/proxy/internal/storage" -) - -var ( - ErrHostedUnsupported = errors.New("pushing packages is not supported for this ecosystem") - ErrHostedInvalidName = errors.New("invalid package name or version") - ErrHostedVersionNotFound = errors.New("no pushed artifacts for this version") - ErrHostedFileNotFound = errors.New("no pushed file of this name for this version") - - // ErrHostedExists refuses a second push of the same file. Clients pin - // pushed files by version and often by hash, so a published file is never - // silently replaced; delete it first to push it again. - ErrHostedExists = errors.New("this file was already pushed; delete it first to push it again") - - // ErrHostedChecksumMismatch reports a push whose bytes don't match the - // sha256 the client sent along. - ErrHostedChecksumMismatch = errors.New("pushed bytes don't match the expected sha256") - - // ErrHostedArtifactUnavailable and ErrHostedArtifactMissing report a - // pushed artifact the cache can't serve. They stop the caller from - // fetching the upstream copy, which would replace the pushed one. - ErrHostedArtifactUnavailable = errors.New("pushed artifact is stored but can't be read") - ErrHostedArtifactMissing = errors.New("pushed artifact is no longer stored") -) - -// HostedTarget is where a pushed archive is stored: the package name and -// version it is cached under and the filename clients request it by. Download -// handlers look artifacts up by exactly these values, so a push under any -// other name would never be served. -type HostedTarget struct { - Name string - Version string - Filename string -} - -// HostedArtifact describes one pushed archive. -type HostedArtifact struct { - Version string `json:"version"` - Filename string `json:"filename"` - SHA256 string `json:"sha256"` - Size int64 `json:"size"` - PushedAt time.Time `json:"pushed_at"` -} - -// hostedEcosystem is what an ecosystem defines for pushed packages. Storing, -// listing, deleting and the push rules are shared; adding an ecosystem means -// adding an entry here and, if its clients read version lists, listing pushed -// versions in its metadata. -type hostedEcosystem struct { - // validName reports whether a package name, split on "/", is one of the - // ecosystem's. - validName func(segments []string) bool - - // filename is the name clients request the archive by, derived from the - // package name and version. When nil, the filename is the last segment of - // the push path, the package name must be a single segment so the path is - // unambiguous, and a version can hold several files (see multiFile). - filename func(name, version string) string - - // readManifest reads the package manifest out of a pushed archive and - // checks that it describes target. When nil, pushes carry no manifest. - readManifest func(archive io.Reader, target HostedTarget) (json.RawMessage, error) - - // hostedOnly reports whether a namespace (an npm scope, a Composer - // vendor) is served from pushed packages alone. When nil, the ecosystem - // has no such namespaces. - hostedOnly func(p *Proxy, namespace string) bool -} - -var hostedEcosystems = map[string]hostedEcosystem{ - "npm": { - validName: func(segments []string) bool { - return len(segments) == 1 || (len(segments) == 2 && strings.HasPrefix(segments[0], "@")) - }, - filename: func(name, version string) string { return lastSegment(name) + "-" + version + ".tgz" }, - readManifest: readNPMManifest, - hostedOnly: func(p *Proxy, scope string) bool { - return slices.Contains(p.HostedNPMScopes, scope) - }, - }, - "composer": { - validName: func(segments []string) bool { return len(segments) == vendorPackageParts }, - filename: func(name, version string) string { return lastSegment(name) + "-" + version + ".zip" }, - readManifest: readComposerManifest, - hostedOnly: func(p *Proxy, vendor string) bool { - return slices.ContainsFunc(p.HostedComposerVendors, func(v string) bool { - return strings.EqualFold(v, vendor) - }) - }, - }, - // generic holds files that aren't packages, such as prebuilt binaries. - // They are served at /generic/hosted/{name}/{version}/{file} and never - // come from an upstream. - genericEcosystem: { - validName: func(segments []string) bool { return len(segments) == 1 }, - }, -} - -// multiFile reports whether a version can hold several files: those named in -// the push path. Such versions get no version-level integrity, which could -// only describe one of them. -func (e hostedEcosystem) multiFile() bool { - return e.filename == nil -} - -// HostedSupported reports whether packages can be pushed for ecosystem. -func HostedSupported(ecosystem string) bool { - _, ok := hostedEcosystems[ecosystem] - return ok -} - -// HostedFileInPath reports whether pushes for ecosystem name the file in the -// path, {name}/{version}/{file}, rather than deriving it from the name and -// version. -func HostedFileInPath(ecosystem string) bool { - eco, ok := hostedEcosystems[ecosystem] - return ok && eco.multiFile() -} - -func lastSegment(name string) string { - return name[strings.LastIndex(name, "/")+1:] -} - -// HostedTargetFor validates a push of name@version to ecosystem and returns -// where it is stored. filename is required for ecosystems whose pushes name -// the file (see HostedFileInPath) and must be empty for the others. -func HostedTargetFor(ecosystem, name, version, filename string) (HostedTarget, error) { - if err := ValidateHostedPath(ecosystem, name, version, filename); err != nil { - return HostedTarget{}, err - } - eco := hostedEcosystems[ecosystem] - if eco.filename != nil { - filename = eco.filename(name, version) - } else if filename == "" { - return HostedTarget{}, ErrHostedInvalidName - } - return HostedTarget{Name: name, Version: version, Filename: filename}, nil -} - -// ValidateHostedPath checks the parts of a hosted API path: a supported -// ecosystem, one of its package names, and path segments that are safe to -// store. filename may be empty, as in a delete of a whole version, but is -// only allowed for ecosystems whose pushes name the file. -func ValidateHostedPath(ecosystem, name, version, filename string) error { - eco, ok := hostedEcosystems[ecosystem] - if !ok { - return fmt.Errorf("%w: %s", ErrHostedUnsupported, ecosystem) - } - segments := strings.Split(name, "/") - if !eco.validName(segments) { - return ErrHostedInvalidName - } - checked := append(slices.Clone(segments), version) - if filename != "" { - if !eco.multiFile() { - return ErrHostedInvalidName - } - checked = append(checked, filename) - } - for _, segment := range checked { - if !validHostedSegment(segment) { - return ErrHostedInvalidName - } - } - return nil -} - -func validHostedSegment(s string) bool { - if s == "" || s == "." || s == ".." || strings.ContainsAny(s, `/\`) { - return false - } - for _, c := range s { - if c < ' ' || c == 0x7f { - return false - } - } - return true -} - -// PushLocalArtifact stores body as the artifact for target and records it as -// local, replacing any cached upstream copy. Download handlers serve it from -// the cache without contacting the upstream, and eviction never removes it. -// A file that was already pushed is refused with ErrHostedExists. When -// wantSHA256 is set, the pushed bytes must hash to it. If the ecosystem has -// manifests, the archive's manifest must name target; it is kept so the -// ecosystem's metadata can list the version with its dependencies. -func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target HostedTarget, body io.Reader, contentType, wantSHA256 string) (HostedArtifact, error) { - if err := p.CheckPush(ecosystem, target); err != nil { - return HostedArtifact{}, err - } - pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, target.Name, target.Version) - if err != nil { - return HostedArtifact{}, err - } - - push := storedPush{ - storagePath: storage.FetchPath(ecosystem, target.Name, target.Version, storage.NewFetchID(), target.Filename), - contentType: contentType, - } - sha512Hash := sha512.New() - storeStart := time.Now() - push.size, push.sha256, err = p.Storage.Store(ctx, push.storagePath, io.TeeReader(body, sha512Hash)) - metrics.RecordStorageOperation("write", time.Since(storeStart)) - if err != nil { - metrics.RecordStorageError("write", metrics.FailureDetail{ - PURL: versionPURL, Filename: target.Filename, Path: push.storagePath, Err: err}) - return HostedArtifact{}, fmt.Errorf("storing artifact: %w", err) - } - push.integrity = "sha512-" + base64.StdEncoding.EncodeToString(sha512Hash.Sum(nil)) - - if wantSHA256 != "" && !strings.EqualFold(wantSHA256, push.sha256) { - p.discardStored(ctx, push.storagePath) - return HostedArtifact{}, fmt.Errorf("%w: got %s", ErrHostedChecksumMismatch, push.sha256) - } - - if err := p.checkPush(ctx, ecosystem, target, versionPURL, &push); err != nil { - p.discardStored(ctx, push.storagePath) - return HostedArtifact{}, err - } - - push.pushedAt = time.Now() - if err := p.recordLocalArtifact(ecosystem, target, pkgPURL, versionPURL, push); err != nil { - p.discardStored(ctx, push.storagePath) - return HostedArtifact{}, err - } - - return HostedArtifact{ - Version: target.Version, - Filename: target.Filename, - SHA256: push.sha256, - Size: push.size, - PushedAt: push.pushedAt.UTC(), - }, nil -} - -// CheckPush refuses a push of a denied version or of a file that was already -// pushed, before any bytes are read, so a client can be told without -// uploading the file. For ecosystems with one file per version, any pushed -// file of the version counts, so a differently cased alias of the name is -// refused too. A cached upstream copy is fine to replace. Recording the push -// checks again atomically, for concurrent pushes. -func (p *Proxy) CheckPush(ecosystem string, target HostedTarget) error { - _, versionPURL, err := packagePURLStrings(ecosystem, target.Name, target.Version) - if err != nil { - return err - } - if p.versionDenied(ecosystem, target.Name, target.Version) { - return fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) - } - var existing []database.Artifact - if hostedEcosystems[ecosystem].multiFile() { - record, err := p.DB.GetArtifact(versionPURL, target.Filename) - if err != nil { - return fmt.Errorf("checking existing artifact: %w", err) - } - if record != nil { - existing = append(existing, *record) - } - } else if existing, err = p.DB.GetArtifactsByVersionPURL(versionPURL); err != nil { - return fmt.Errorf("checking existing artifacts: %w", err) - } - for _, a := range existing { - if a.Source == database.SourceLocal { - return ErrHostedExists - } - } - return nil -} - -// storedPush is a pushed archive once it is in storage. -type storedPush struct { - storagePath string - sha256 string - integrity string - size int64 - contentType string - manifest json.RawMessage - pushedAt time.Time -} - -// checkPush reads the manifest of a stored push and runs the scanners on it. -func (p *Proxy) checkPush(ctx context.Context, ecosystem string, target HostedTarget, versionPURL string, push *storedPush) error { - if hostedEcosystems[ecosystem].readManifest != nil { - stored, err := p.Storage.Open(ctx, push.storagePath) - if err != nil { - return fmt.Errorf("reading pushed archive: %w", err) - } - push.manifest, err = readHostedManifest(ecosystem, stored, target) - _ = stored.Close() - if err != nil { - return err - } - } - - if p.Scanners != nil && p.Scanners.Enabled() { - return p.runScan(ctx, ecosystem, target.Name, target.Version, target.Filename, versionPURL, push.storagePath, push.size, push.contentType) - } - return nil -} - -func (p *Proxy) discardStored(ctx context.Context, storagePath string) { - if err := p.Storage.Delete(context.WithoutCancel(ctx), storagePath); err != nil { - p.Logger.Warn("failed to delete pushed artifact from storage", "path", storagePath, "error", err) - } -} - -// recordLocalArtifact writes the package, version and artifact records in -// one transaction. An existing package record is kept as it is, since it may -// describe the upstream package. For ecosystems with one file per version, -// the version's integrity is set to the pushed bytes, so the cache's -// integrity check verifies them rather than an upstream archive's digest. -// Multi-file versions can't have one integrity, so an existing version record -// is left alone and each file is checked against its own sha256. -func (p *Proxy) recordLocalArtifact(ecosystem string, target HostedTarget, pkgPURL, versionPURL string, push storedPush) error { - oneFilePerVersion := !hostedEcosystems[ecosystem].multiFile() - err := p.DB.RecordLocalArtifact( - &database.Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: target.Name}, - &database.Version{ - PURL: versionPURL, - PackagePURL: pkgPURL, - Integrity: sql.NullString{String: push.integrity, Valid: oneFilePerVersion}, - }, - oneFilePerVersion, - &database.Artifact{ - VersionPURL: versionPURL, - Filename: target.Filename, - StoragePath: sql.NullString{String: push.storagePath, Valid: true}, - ContentHash: sql.NullString{String: push.sha256, Valid: true}, - Size: sql.NullInt64{Int64: push.size, Valid: true}, - ContentType: sql.NullString{String: push.contentType, Valid: push.contentType != ""}, - FetchedAt: sql.NullTime{Time: push.pushedAt, Valid: true}, - Manifest: sql.NullString{String: string(push.manifest), Valid: push.manifest != nil}, - }) - if errors.Is(err, database.ErrLocalArtifactExists) { - return ErrHostedExists - } - if err != nil { - return fmt.Errorf("recording pushed artifact: %w", err) - } - return nil -} - -// LocalArtifacts lists the archives pushed for a package. -func (p *Proxy) LocalArtifacts(ecosystem, name string) ([]HostedArtifact, error) { - pkgPURL := packageurl.MakeString(ecosystem, name, "") - if pkgPURL == "" { - return nil, fmt.Errorf("%w: %s %q", errUnsupportedPackageIdentity, ecosystem, name) - } - rows, err := p.DB.GetLocalArtifacts(pkgPURL) - if err != nil { - return nil, fmt.Errorf("listing local artifacts: %w", err) - } - listed := make([]HostedArtifact, 0, len(rows)) - for _, row := range rows { - listed = append(listed, HostedArtifact{ - Version: database.VersionFromPURL(row.VersionPURL), - Filename: row.Filename, - SHA256: row.ContentHash.String, - Size: row.Size.Int64, - PushedAt: row.FetchedAt.Time.UTC(), - }) - } - return listed, nil -} - -// hostedOnly reports whether a package is in an npm scope or Composer vendor -// that is served from pushed packages alone. The upstream is never asked -// about such a package, so nobody can mix in versions by publishing the same -// name there. -func (p *Proxy) hostedOnly(ecosystem, name string) bool { - if !p.HostedEnabled { - return false - } - namespace, _, namespaced := strings.Cut(name, "/") - eco := hostedEcosystems[ecosystem] - return namespaced && eco.hostedOnly != nil && eco.hostedOnly(p, namespace) -} - -// localVersion is a pushed version as ecosystem metadata lists it. -type localVersion struct { - Version string - Filename string - SHA256 string - Integrity string - Manifest json.RawMessage - PushedAt time.Time -} - -// localVersions returns the versions pushed for a package, or nil when hosted -// packages are disabled, so metadata requests cost no extra lookup then. -// Denied versions are left out, so metadata never lists a version whose -// download is refused. -func (p *Proxy) localVersions(ecosystem, name string) ([]localVersion, error) { - if !p.HostedEnabled { - return nil, nil - } - pkgPURL := packageurl.MakeString(ecosystem, name, "") - if pkgPURL == "" { - return nil, nil - } - rows, err := p.DB.GetLocalArtifacts(pkgPURL) - if err != nil { - return nil, fmt.Errorf("listing local versions: %w", err) - } - versions := make([]localVersion, 0, len(rows)) - for _, row := range rows { - version := database.VersionFromPURL(row.VersionPURL) - if !row.Manifest.Valid || p.versionDenied(ecosystem, name, version) { - continue - } - versions = append(versions, localVersion{ - Version: version, - Filename: row.Filename, - SHA256: row.ContentHash.String, - Integrity: row.Integrity.String, - Manifest: json.RawMessage(row.Manifest.String), - PushedAt: row.FetchedAt.Time.UTC(), - }) - } - return versions, nil -} - -// DeleteLocalVersion removes the archives pushed for name@version. If the -// upstream has the version, clients get the upstream archive again. -func (p *Proxy) DeleteLocalVersion(ecosystem, name, version string) error { - return p.deleteLocal(ecosystem, name, version, "", ErrHostedVersionNotFound) -} - -// DeleteLocalFile removes one pushed file of name@version. -func (p *Proxy) DeleteLocalFile(ecosystem, name, version, filename string) error { - return p.deleteLocal(ecosystem, name, version, filename, ErrHostedFileNotFound) -} - -func (p *Proxy) deleteLocal(ecosystem, name, version, filename string, notFound error) error { - _, versionPURL, err := packagePURLStrings(ecosystem, name, version) - if err != nil { - return err - } - deleted, err := p.DB.DeleteLocalArtifacts(versionPURL, filename) - if err != nil { - return fmt.Errorf("deleting local artifacts: %w", err) - } - if deleted == 0 { - return notFound - } - return nil -} - -// GetLocalArtifact serves a pushed artifact from storage, or returns nil -// when the artifact was not pushed. Handlers that must consult the upstream -// before serving a cached copy call it first, so pushed versions the upstream -// does not list are still served. Pushed artifacts are not a cache, so they -// are served even when StreamArtifacts is set. -// -// With hosted packages disabled it returns nil without a lookup, as -// localVersions does: pushes are then left out of metadata, so they are not -// served ahead of the upstream either. -func (p *Proxy) GetLocalArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { - if !p.HostedEnabled { - return nil, nil - } - pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) - if err != nil { - return nil, err - } - record, err := p.DB.GetArtifact(versionPURL, filename) - if err != nil { - return nil, fmt.Errorf("checking artifact source: %w", err) - } - if record == nil || record.Source != database.SourceLocal { - return nil, nil - } - if p.Denylist.Denied(versionPURL) { - return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) - } - result, err := p.readStored(ctx, pkgPURL, versionPURL, filename) - if err != nil || result != nil { - return result, err - } - return nil, p.localArtifactMiss(versionPURL, filename) -} diff --git a/internal/handler/hosted_manifest.go b/internal/handler/hosted_manifest.go deleted file mode 100644 index cf90dcb..0000000 --- a/internal/handler/hosted_manifest.go +++ /dev/null @@ -1,170 +0,0 @@ -package handler - -import ( - "archive/tar" - "archive/zip" - "bytes" - "compress/gzip" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path" - "strings" -) - -var ( - ErrHostedInvalidArchive = errors.New("archive has no readable package manifest") - ErrHostedManifestMismatch = errors.New("package manifest does not match the pushed name and version") -) - -// maxManifestSize bounds how much of a manifest file is read. Real manifests -// are a few kilobytes; the bound keeps a hostile archive from claiming memory. -const maxManifestSize = 1 << 20 - -// readHostedManifest reads the package manifest out of a pushed archive and -// checks that it describes target. The manifest is what the ecosystem's -// metadata lists the version with, so its dependencies reach the client. -// Ecosystems without manifests return nil. -func readHostedManifest(ecosystem string, archive io.Reader, target HostedTarget) (json.RawMessage, error) { - eco, ok := hostedEcosystems[ecosystem] - if !ok { - return nil, fmt.Errorf("%w: %s", ErrHostedUnsupported, ecosystem) - } - if eco.readManifest == nil { - return nil, nil - } - return eco.readManifest(archive, target) -} - -// readNPMManifest reads package.json from the top directory of an npm -// tarball. npm packs it under package/, but published tarballs use other -// directory names too. -func readNPMManifest(archive io.Reader, target HostedTarget) (json.RawMessage, error) { - gz, err := gzip.NewReader(archive) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) - } - defer func() { _ = gz.Close() }() - - tr := tar.NewReader(gz) - for { - header, err := tr.Next() - if err != nil { - return nil, fmt.Errorf("%w: no package.json in tarball", ErrHostedInvalidArchive) - } - if header.Typeflag != tar.TypeReg || !isTopLevelManifest(header.Name, "package.json") { - continue - } - manifest, fields, err := decodeManifest(tr) - if err != nil { - return nil, err - } - if fields.Name != target.Name || fields.Version != target.Version { - return nil, fmt.Errorf("%w: package.json is %s@%s", ErrHostedManifestMismatch, fields.Name, fields.Version) - } - return manifest, nil - } -} - -// readComposerManifest reads composer.json from the root of a Composer dist -// zip, or from its single top directory as GitHub zipballs have it. Package -// names are case-insensitive, and composer.json only names a version when -// the package pins one. -func readComposerManifest(archive io.Reader, target HostedTarget) (json.RawMessage, error) { - file, err := os.CreateTemp("", "hosted-composer-*.zip") - if err != nil { - return nil, err - } - defer func() { - _ = file.Close() - _ = os.Remove(file.Name()) - }() - size, err := io.Copy(file, archive) - if err != nil { - return nil, err - } - zr, err := zip.NewReader(file, size) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) - } - - entry, err := composerManifestEntry(zr.File) - if err != nil { - return nil, err - } - rc, err := entry.Open() - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) - } - manifest, fields, err := decodeManifest(rc) - _ = rc.Close() - if err != nil { - return nil, err - } - if !strings.EqualFold(fields.Name, target.Name) || (fields.Version != "" && fields.Version != target.Version) { - return nil, fmt.Errorf("%w: composer.json is %s %s", ErrHostedManifestMismatch, fields.Name, fields.Version) - } - return manifest, nil -} - -// composerManifestEntry picks the package's composer.json: the one at the -// root, or else the only one directly inside a top directory. Archives keep -// other composer.json files for tests and examples, in any order, so the -// first one found isn't necessarily the package's. -func composerManifestEntry(files []*zip.File) (*zip.File, error) { - var topLevel []*zip.File - for _, entry := range files { - if entry.FileInfo().IsDir() { - continue - } - if entry.Name == "composer.json" { - return entry, nil - } - if isTopLevelManifest(entry.Name, "composer.json") { - topLevel = append(topLevel, entry) - } - } - switch len(topLevel) { - case 0: - return nil, fmt.Errorf("%w: no composer.json in zip", ErrHostedInvalidArchive) - case 1: - return topLevel[0], nil - default: - return nil, fmt.Errorf("%w: no composer.json at the root and %d in top directories", ErrHostedInvalidArchive, len(topLevel)) - } -} - -// isTopLevelManifest reports whether name is filename inside one top -// directory, such as package/package.json. -func isTopLevelManifest(name, filename string) bool { - dir, file := path.Split(path.Clean(name)) - return file == filename && dir != "" && !strings.Contains(strings.TrimSuffix(dir, "/"), "/") -} - -type manifestFields struct { - Name string `json:"name"` - Version string `json:"version"` -} - -// decodeManifest reads a JSON object manifest and returns it compacted along -// with its name and version. -func decodeManifest(r io.Reader) (json.RawMessage, manifestFields, error) { - raw, err := io.ReadAll(io.LimitReader(r, maxManifestSize+1)) - if err != nil { - return nil, manifestFields{}, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) - } - if len(raw) > maxManifestSize { - return nil, manifestFields{}, fmt.Errorf("%w: manifest exceeds %d bytes", ErrHostedInvalidArchive, maxManifestSize) - } - var fields manifestFields - if err := json.Unmarshal(raw, &fields); err != nil { - return nil, manifestFields{}, fmt.Errorf("%w: manifest is not a JSON object: %w", ErrHostedInvalidArchive, err) - } - var compact bytes.Buffer - if err := json.Compact(&compact, raw); err != nil { - return nil, manifestFields{}, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) - } - return compact.Bytes(), fields, nil -} diff --git a/internal/handler/hosted_test.go b/internal/handler/hosted_test.go deleted file mode 100644 index b0640be..0000000 --- a/internal/handler/hosted_test.go +++ /dev/null @@ -1,562 +0,0 @@ -package handler - -import ( - "archive/tar" - "archive/zip" - "bytes" - "compress/gzip" - "context" - "errors" - "fmt" - "io" - "strings" - "sync" - "testing" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/registries/fetch" -) - -func TestHostedTargetForFileInPath(t *testing.T) { - got, err := HostedTargetFor("generic", "tools", "1.2.0", "tool.tar.gz") - if err != nil || got != (HostedTarget{Name: "tools", Version: "1.2.0", Filename: "tool.tar.gz"}) { - t.Errorf("generic target = %+v, %v", got, err) - } - for _, tt := range []struct{ ecosystem, name, version, file string }{ - {"generic", "tools", "1.2.0", ""}, - {"generic", "tools", "1.2.0", ".."}, - {"generic", "acme/tools", "1.2.0", "tool.tar.gz"}, - {"npm", "request", "2.88.3", "request-2.88.3.tgz"}, - } { - if _, err := HostedTargetFor(tt.ecosystem, tt.name, tt.version, tt.file); !errors.Is(err, ErrHostedInvalidName) { - t.Errorf("HostedTargetFor(%q, %q, %q, %q) error = %v, want ErrHostedInvalidName", - tt.ecosystem, tt.name, tt.version, tt.file, err) - } - } - if !HostedFileInPath("generic") || HostedFileInPath("npm") || HostedFileInPath("cargo") { - t.Error("HostedFileInPath should hold for generic only") - } -} - -func TestHostedTargetFor(t *testing.T) { - tests := []struct { - name string - ecosystem, pkg, version string - want HostedTarget - wantErr error - }{ - {name: "npm", ecosystem: "npm", pkg: "request", version: "2.88.3", - want: HostedTarget{Name: "request", Version: "2.88.3", Filename: "request-2.88.3.tgz"}}, - {name: "npm scoped", ecosystem: "npm", pkg: "@types/node", version: "20.1.0", - want: HostedTarget{Name: "@types/node", Version: "20.1.0", Filename: "node-20.1.0.tgz"}}, - {name: "composer", ecosystem: "composer", pkg: "monolog/monolog", version: "3.5.0", - want: HostedTarget{Name: "monolog/monolog", Version: "3.5.0", Filename: "monolog-3.5.0.zip"}}, - {name: "composer without vendor", ecosystem: "composer", pkg: "monolog", version: "3.5.0", - wantErr: ErrHostedInvalidName}, - {name: "composer with extra segment", ecosystem: "composer", pkg: "a/b/c", version: "1.0.0", - wantErr: ErrHostedInvalidName}, - {name: "unsupported ecosystem", ecosystem: "cargo", pkg: "serde", version: "1.0.200", - wantErr: ErrHostedUnsupported}, - {name: "unknown ecosystem", ecosystem: "maven", pkg: "junit", version: "4.13", - wantErr: ErrHostedUnsupported}, - {name: "npm scope without @", ecosystem: "npm", pkg: "types/node", version: "1.0.0", - wantErr: ErrHostedInvalidName}, - {name: "dot-dot name segment", ecosystem: "npm", pkg: "@scope/..", version: "1.0.0", - wantErr: ErrHostedInvalidName}, - {name: "dot-dot version", ecosystem: "npm", pkg: "request", version: "..", - wantErr: ErrHostedInvalidName}, - {name: "control character", ecosystem: "npm", pkg: "request", version: "1.0.0\n", - wantErr: ErrHostedInvalidName}, - {name: "empty version", ecosystem: "npm", pkg: "request", version: "", - wantErr: ErrHostedInvalidName}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := HostedTargetFor(tt.ecosystem, tt.pkg, tt.version, "") - if tt.wantErr != nil { - if !errors.Is(err, tt.wantErr) { - t.Fatalf("HostedTargetFor() error = %v, want %v", err, tt.wantErr) - } - return - } - if err != nil { - t.Fatalf("HostedTargetFor() error = %v", err) - } - if got != tt.want { - t.Errorf("HostedTargetFor() = %+v, want %+v", got, tt.want) - } - }) - } -} - -// npmTarball builds an npm package tarball holding manifest as -// package/package.json. Building an archive in memory cannot fail, so errors -// panic. -func npmTarball(manifest string) []byte { - var buf bytes.Buffer - gz := gzip.NewWriter(&buf) - tw := tar.NewWriter(gz) - for _, f := range [][2]string{{"package/package.json", manifest}, {"package/index.js", "module.exports = 1\n"}} { - must(tw.WriteHeader(&tar.Header{Name: f[0], Mode: 0o644, Size: int64(len(f[1]))})) - _, err := io.WriteString(tw, f[1]) - must(err) - } - must(tw.Close()) - must(gz.Close()) - return buf.Bytes() -} - -// composerZip builds a Composer dist archive holding manifest as composer.json -// under dir, or at the root when dir is empty. -func composerZip(dir, manifest string) []byte { - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - prefix := "" - if dir != "" { - prefix = dir + "/" - } - for _, f := range [][2]string{{prefix + "composer.json", manifest}, {prefix + "src/Logger.php", "=8.1"}}`)}, - {name: "composer manifest in a top directory", ecosystem: "composer", pkg: "monolog/monolog", version: "3.5.0", - archive: composerZip("Seldaek-monolog-abc123", `{"name":"Monolog/Monolog","version":"3.5.0"}`)}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, _, store, _ := setupTestProxy(t) - target, err := HostedTargetFor(tt.ecosystem, tt.pkg, tt.version, "") - if err != nil { - t.Fatalf("HostedTargetFor: %v", err) - } - _, err = proxy.PushLocalArtifact(context.Background(), tt.ecosystem, target, bytes.NewReader(tt.archive), "", "") - if tt.wantErr == nil { - if err != nil { - t.Fatalf("PushLocalArtifact: %v", err) - } - return - } - if !errors.Is(err, tt.wantErr) { - t.Fatalf("PushLocalArtifact error = %v, want %v", err, tt.wantErr) - } - if listed, _ := proxy.LocalArtifacts(tt.ecosystem, tt.pkg); len(listed) != 0 { - t.Errorf("rejected archive was recorded: %+v", listed) - } - if len(store.files) != 0 { - t.Errorf("rejected archive left %d objects in storage", len(store.files)) - } - }) - } -} - -// removeStored deletes the stored object of an artifact behind the cache's back. -func removeStored(t *testing.T, proxy *Proxy, versionPURL, filename string) { - t.Helper() - record, err := proxy.DB.GetArtifact(versionPURL, filename) - if err != nil || record == nil { - t.Fatalf("GetArtifact: %+v, %v", record, err) - } - if err := proxy.Storage.Delete(context.Background(), record.StoragePath.String); err != nil { - t.Fatalf("deleting stored object: %v", err) - } -} - -func TestPushedArtifactIsNeverReplacedFromUpstream(t *testing.T) { - const versionPURL, filename = "pkg:npm/request@2.88.3", "request-2.88.3.tgz" - tests := []struct { - name string - damage func(t *testing.T, proxy *Proxy) - wantErr error - }{ - {name: "stored object missing", wantErr: ErrHostedArtifactUnavailable, - damage: func(t *testing.T, proxy *Proxy) { removeStored(t, proxy, versionPURL, filename) }}, - {name: "record cleared after an integrity failure", wantErr: ErrHostedArtifactMissing, - damage: func(t *testing.T, proxy *Proxy) { - record, err := proxy.DB.GetArtifact(versionPURL, filename) - if err != nil || record == nil { - t.Fatalf("GetArtifact: %+v, %v", record, err) - } - if err := proxy.DB.DiscardArtifact(versionPURL, filename, record.StoragePath.String); err != nil { - t.Fatalf("DiscardArtifact: %v", err) - } - }}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, db, _, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream tarball")), ContentType: "application/gzip"} - pushRequest(t, proxy) - tt.damage(t, proxy) - - _, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "request", "2.88.3", filename, - "https://registry.npmjs.org/request/-/request-2.88.3.tgz") - if !errors.Is(err, tt.wantErr) { - t.Errorf("GetOrFetchArtifactFromURL error = %v, want %v", err, tt.wantErr) - } - if fetcher.fetchCalled { - t.Error("a miss on a pushed artifact fetched the upstream copy") - } - local, err := db.GetLocalArtifacts("pkg:npm/request") - if err != nil || len(local) != 1 || !local[0].Manifest.Valid { - t.Errorf("pushed records after the miss = %+v, %v, want the pushed one with its manifest", local, err) - } - }) - } -} - -// zipWith builds a zip holding files in the given order. -func zipWith(files ...[2]string) []byte { - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - for _, f := range files { - w, err := zw.Create(f[0]) - must(err) - _, err = io.WriteString(w, f[1]) - must(err) - } - must(zw.Close()) - return buf.Bytes() -} - -func TestComposerManifestLookup(t *testing.T) { - const monolog = `{"name":"monolog/monolog"}` - const fixture = `{"name":"acme/test-fixture"}` - tests := []struct { - name string - archive []byte - wantErr error - }{ - {name: "root manifest after a nested one", - archive: zipWith([2]string{"tests/composer.json", fixture}, [2]string{"composer.json", monolog})}, - {name: "top directory manifest next to a deeper one", - archive: zipWith([2]string{"monolog-abc/tests/composer.json", fixture}, [2]string{"monolog-abc/composer.json", monolog})}, - {name: "two top directories with manifests", - archive: zipWith([2]string{"tests/composer.json", fixture}, [2]string{"monolog-abc/composer.json", monolog}), - wantErr: ErrHostedInvalidArchive}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - target, err := HostedTargetFor("composer", "monolog/monolog", "3.5.0", "") - if err != nil { - t.Fatalf("HostedTargetFor: %v", err) - } - _, err = readHostedManifest("composer", bytes.NewReader(tt.archive), target) - if tt.wantErr == nil && err != nil { - t.Errorf("readHostedManifest error = %v, want the root or single top directory manifest", err) - } - if tt.wantErr != nil && !errors.Is(err, tt.wantErr) { - t.Errorf("readHostedManifest error = %v, want %v", err, tt.wantErr) - } - }) - } -} - -func TestConcurrentPushesOfTheSameFileHaveOneWinner(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.HostedEnabled = true - target, err := HostedTargetFor("generic", "tools", "1.2.0", "tool.tar.gz") - if err != nil { - t.Fatalf("HostedTargetFor: %v", err) - } - - const pushes = 8 - start := make(chan struct{}) - results := make([]error, pushes) - pushed := make([]HostedArtifact, pushes) - var wg sync.WaitGroup - for i := range pushes { - wg.Add(1) - go func() { - defer wg.Done() - <-start - body := strings.NewReader(fmt.Sprintf("build %d", i)) - pushed[i], results[i] = proxy.PushLocalArtifact(context.Background(), "generic", target, body, "", "") - }() - } - close(start) - wg.Wait() - - winner := -1 - for i, err := range results { - switch { - case err == nil && winner >= 0: - t.Fatalf("pushes %d and %d both succeeded", winner, i) - case err == nil: - winner = i - case !errors.Is(err, ErrHostedExists): - t.Fatalf("push %d error = %v, want ErrHostedExists", i, err) - } - } - if winner < 0 { - t.Fatal("no push succeeded") - } - - result, err := proxy.GetLocalArtifact(context.Background(), "generic", "tools", "1.2.0", "tool.tar.gz") - if err != nil || result == nil { - t.Fatalf("GetLocalArtifact = %v, %v", result, err) - } - served, _ := io.ReadAll(result.Reader) - _ = result.Reader.Close() - if string(served) != fmt.Sprintf("build %d", winner) { - t.Errorf("served %q, want the winning push %q", served, fmt.Sprintf("build %d", winner)) - } -} - -// A fetch that started before a push and records after it must not replace -// the pushed artifact or its version's integrity. -func TestCacheFillAfterPushKeepsThePush(t *testing.T) { - proxy, _, store, _ := setupTestProxy(t) - proxy.HostedEnabled = true - pushRequest(t, proxy) - - const fetchedPath = "npm/request/2.88.3/fetched/request-2.88.3.tgz" - if _, _, err := store.Store(context.Background(), fetchedPath, strings.NewReader("upstream bytes")); err != nil { - t.Fatalf("storing fetched copy: %v", err) - } - fetched := testArtifact("upstream bytes", "pkg:npm/request@2.88.3", "request-2.88.3.tgz", "application/gzip") - err := proxy.updateCacheDB("npm", "request", "pkg:npm/request", "https://registry.npmjs.org/request/-/request-2.88.3.tgz", - fetchedPath, fetched, "sha512-upstream") - if !errors.Is(err, database.ErrLocalArtifactExists) { - t.Fatalf("updateCacheDB over a push error = %v, want ErrLocalArtifactExists", err) - } - - result, err := proxy.GetLocalArtifact(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz") - if err != nil || result == nil { - t.Fatalf("GetLocalArtifact = %v, %v, want the pushed copy", result, err) - } - served, err := io.ReadAll(result.Reader) - _ = result.Reader.Close() - if err != nil || string(served) != pushedTarball { - t.Errorf("served %d bytes, %v, want the pushed tarball, integrity intact", len(served), err) - } -} - -func TestGenericPushDoesNotReadTheStoredFile(t *testing.T) { - proxy, _, store, _ := setupTestProxy(t) - proxy.HostedEnabled = true - target, err := HostedTargetFor("generic", "tools", "1.2.0", "tool.tar.gz") - if err != nil { - t.Fatalf("HostedTargetFor: %v", err) - } - store.openErr = errors.New("storage opened") - if _, err := proxy.PushLocalArtifact(context.Background(), "generic", target, strings.NewReader("bytes"), "", ""); err != nil { - t.Fatalf("PushLocalArtifact: %v", err) - } -} diff --git a/internal/handler/if_none_match_test.go b/internal/handler/if_none_match_test.go deleted file mode 100644 index b659254..0000000 --- a/internal/handler/if_none_match_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package handler - -import "testing" - -func TestIfNoneMatchHits(t *testing.T) { - tests := []struct { - header string - etag string - want bool - }{ - {`"abc"`, `"abc"`, true}, - {`"abc"`, `"def"`, false}, - {"", `"abc"`, false}, - {`"abc"`, "", false}, - {"*", `"abc"`, true}, - {"*", "", false}, - {`W/"abc"`, `"abc"`, true}, - {`"abc"`, `W/"abc"`, true}, - {`W/"abc"`, `W/"abc"`, true}, - {`"abc", "def"`, `"def"`, true}, - {`"abc","def"`, `"def"`, true}, - {` "abc" , W/"def" `, `"def"`, true}, - {`"abc", "def"`, `"ghi"`, false}, - } - - for _, tt := range tests { - if got := ifNoneMatchHits(tt.header, tt.etag); got != tt.want { - t.Errorf("ifNoneMatchHits(%q, %q) = %v, want %v", tt.header, tt.etag, got, tt.want) - } - } -} diff --git a/internal/handler/integrity.go b/internal/handler/integrity.go index 1bbd406..bb29a21 100644 --- a/internal/handler/integrity.go +++ b/internal/handler/integrity.go @@ -1,297 +1,140 @@ package handler import ( + "crypto/sha256" + "crypto/sha512" + "crypto/subtle" + "encoding/base64" + "encoding/hex" "fmt" + "hash" "io" - - "github.com/git-pkgs/integrity" + "strings" ) -type integrityChecks struct { - contentHash integrity.SRI - native integrity.SRI - // checksum is native metadata given as a single algorithm-prefixed hex - // checksum, such as a registry's SHA-1 shasum, which is not SRI. - checksum integrity.Digest - hasChecksum bool - algorithms []integrity.Algorithm -} - -func newIntegrityChecks(contentHash, native string) (integrityChecks, error) { - checks := integrityChecks{} - - if contentHash != "" { - digest, err := integrity.ParseHex(integrity.SHA256, contentHash) - if err != nil { - return integrityChecks{}, fmt.Errorf("parse content_hash: %w", err) - } - checks.contentHash = integrity.SRI{digest} - checks.algorithms = append(checks.algorithms, integrity.SHA256) +// parseSRI parses a Subresource Integrity string (e.g. "sha512-abc==") into +// an algorithm name and raw digest bytes. Returns ok=false for empty, +// malformed, or unsupported entries. Only the first hash in a multi-hash +// SRI string is considered. +func parseSRI(s string) (algo string, digest []byte, ok bool) { + s = strings.TrimSpace(s) + if s == "" { + return "", nil, false } - - if native != "" { - digests, err := integrity.ParseSRI(native) - if err != nil { - checksum, checksumErr := integrity.ParseChecksum(native) - if checksumErr != nil { - return integrityChecks{}, fmt.Errorf("parse integrity: %w", err) - } - checks.checksum, checks.hasChecksum = checksum, true - checks.algorithms = append(checks.algorithms, checksum.Algorithm()) - return checks, nil - } - checks.native = digests - for _, digest := range digests { - checks.algorithms = append(checks.algorithms, digest.Algorithm()) - } + if i := strings.IndexByte(s, ' '); i >= 0 { + s = s[:i] } - - return checks, nil -} - -// forStoredObject returns the checks to run when serving a stored object. -// When the content hash is recorded it alone is checked. It was computed -// from the bytes that were verified against the native integrity before -// they were stored, so a match binds the bytes just as well, and serving a -// cache hit costs one hash instead of two (SHA-256 plus SHA-512 for npm). -// Without a content hash the native digests are all there is to check. -func (c integrityChecks) forStoredObject() integrityChecks { - if len(c.contentHash) == 0 { - return c + algo, b64, found := strings.Cut(s, "-") + if !found { + return "", nil, false } - return integrityChecks{contentHash: c.contentHash, algorithms: []integrity.Algorithm{integrity.SHA256}} -} - -// verifyIntegrity wraps body to check it against the declared integrity as -// it is read. verified, called once body has been read, reports a mismatch as -// ErrArtifactDigestMismatch, and also a body that was not read to its end, -// since only a complete body can be checked. An empty declared checks -// nothing. -func verifyIntegrity(body io.ReadCloser, declared string) (_ io.Reader, verified func() error, _ error) { - if declared == "" { - return body, func() error { return nil }, nil - } - checks, err := newIntegrityChecks("", declared) + d, err := base64.StdEncoding.DecodeString(b64) if err != nil { - return nil, nil, err + return "", nil, false } - var mismatch string - reader, err := checks.newVerifyingReader(body, func(reason string) { mismatch = reason }, false) - if err != nil { - return nil, nil, err + switch algo { + case "sha256", "sha384", "sha512": + return algo, d, true + default: + return "", nil, false } - return reader, func() error { - if !reader.(*verifyingReader).verified { - return fmt.Errorf("%w: body was not read to its end", ErrArtifactDigestMismatch) - } - if mismatch != "" { - return fmt.Errorf("%w: %s", ErrArtifactDigestMismatch, mismatch) - } - return nil - }, nil } -func (c integrityChecks) wrap(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) { - return c.wrapSeekable(source, onMismatch, false) +func newSRIHash(algo string) hash.Hash { + switch algo { + case "sha256": + return sha256.New() + case "sha384": + return sha512.New384() + case "sha512": + return sha512.New() + } + return nil } -// wrapFailOnMismatch is wrap for bytes that are served to a client: the -// final chunk is held back until the digest is checked, and a mismatch drops -// it and is returned from Read as ErrArtifactDigestMismatch in place of -// io.EOF, so the caller can abort rather than complete the response. -func (c integrityChecks) wrapFailOnMismatch(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) { - return c.wrapSeekable(source, onMismatch, true) -} - -func (c integrityChecks) wrapSeekable(source io.ReadCloser, onMismatch func(string), failOnMismatch bool) (io.ReadCloser, error) { - if len(c.algorithms) == 0 { - return source, nil - } - verified, err := c.newVerifyingReader(source, onMismatch, failOnMismatch) - if err != nil { - return nil, err - } - seeker, ok := source.(io.Seeker) - if !ok { - return verified, nil - } - return &seekableVerifyingReader{ - source: source, - verified: verified, - seeker: seeker, - }, nil -} - -func (c integrityChecks) newVerifyingReader(source io.ReadCloser, onMismatch func(string), failOnMismatch bool) (io.ReadCloser, error) { - if len(c.algorithms) == 0 { - return source, nil - } - reader, err := integrity.NewReader(source, c.algorithms...) - if err != nil { - return nil, fmt.Errorf("create integrity reader: %w", err) - } - return &verifyingReader{ - source: source, - reader: reader, - checks: c, - onMismatch: onMismatch, - failOnMismatch: failOnMismatch, - }, nil -} - -// verifyingReader forwards Close to its source and reports completed digest -// mismatches after its shared integrity reader observes EOF. +// verifyingReader wraps an io.ReadCloser and computes SHA256 (and optionally +// a second SRI hash) as bytes are read. When the underlying reader reaches +// EOF it compares the digests against the expected values and calls +// onMismatch for each failure. Verification is skipped if the stream was +// not fully consumed (e.g. client disconnect) to avoid false positives. type verifyingReader struct { - source io.ReadCloser - reader *integrity.Reader - checks integrityChecks + r io.ReadCloser + sha256 hash.Hash + wantSHA256 string + sri hash.Hash + sriAlgo string + wantSRI []byte onMismatch func(reason string) - verified bool - - failOnMismatch bool - mismatched bool - - // With failOnMismatch, held (backed by buf) is read but not yet - // returned, and only its first releasable bytes are known not to end the - // body. The rest is the latest chunk, returned once the digest matches. - buf []byte - held []byte - releasable int eof bool + verified bool } -// seekableVerifyingReader verifies ordinary reads until a successful seek, -// after which reads bypass whole-object verification for range responses. -type seekableVerifyingReader struct { - source io.ReadCloser - verified io.ReadCloser - seeker io.Seeker - bypass bool -} - -func (r *seekableVerifyingReader) Read(p []byte) (int, error) { - if r.bypass { - return r.source.Read(p) +func newVerifyingReader(r io.ReadCloser, contentHash, sri string, onMismatch func(string)) io.ReadCloser { + if contentHash == "" && sri == "" { + return r } - return r.verified.Read(p) -} - -func (r *seekableVerifyingReader) Seek(offset int64, whence int) (int64, error) { - position, err := r.seeker.Seek(offset, whence) - if err == nil { - r.bypass = true + v := &verifyingReader{ + r: r, + onMismatch: onMismatch, } - return position, err -} - -func (r *seekableVerifyingReader) Close() error { - return r.verified.Close() -} - -func (r *verifyingReader) Read(p []byte) (int, error) { - if r.failOnMismatch { - return r.readHoldingBack(p) + if contentHash != "" { + v.sha256 = sha256.New() + v.wantSHA256 = contentHash + } + if algo, digest, ok := parseSRI(sri); ok { + v.sri = newSRIHash(algo) + v.sriAlgo = algo + v.wantSRI = digest + } + if v.sha256 == nil && v.sri == nil { + return r + } + return v +} + +func (v *verifyingReader) Read(p []byte) (int, error) { + n, err := v.r.Read(p) + if n > 0 { + if v.sha256 != nil { + v.sha256.Write(p[:n]) + } + if v.sri != nil { + v.sri.Write(p[:n]) + } } - n, err := r.reader.Read(p) if err == io.EOF { - r.verify() + v.eof = true + v.verify() } return n, err } -// readHoldingBack returns bytes one chunk behind the source, so the final -// chunk is released only after the digest over the whole body has matched. -func (r *verifyingReader) readHoldingBack(p []byte) (int, error) { - if len(p) == 0 { - return 0, nil +func (v *verifyingReader) Close() error { + if v.eof { + v.verify() } - for r.releasable == 0 && !r.eof { - // Move the held chunk to the front of the buffer and read the next - // chunk after it; once that arrives, the held chunk is not the last. - var pending int - if cap(r.buf) < len(r.held)+len(p) { - buf := make([]byte, len(r.held)+len(p)) - pending = copy(buf, r.held) - r.buf = buf - } else { - pending = copy(r.buf[:cap(r.buf)], r.held) - } - n, err := r.reader.Read(r.buf[pending : pending+len(p)]) - r.held = r.buf[:pending+n] - if n > 0 { - r.releasable = pending - } - switch { - case err == io.EOF: - r.verify() - r.eof = true - case err != nil: - r.held, r.releasable = nil, 0 - return 0, err - case n == 0: - return 0, nil - } - } - if r.eof { - if r.mismatched { - r.held, r.releasable = nil, 0 - return 0, ErrArtifactDigestMismatch - } - r.releasable = len(r.held) - if r.releasable == 0 { - return 0, io.EOF - } - } - n := copy(p, r.held[:r.releasable]) - r.held = r.held[n:] - r.releasable -= n - return n, nil + return v.r.Close() } -func (r *verifyingReader) Close() error { - return r.source.Close() -} - -func (r *verifyingReader) verify() { - if r.verified { - return - } - r.verified = true - result := r.reader.Result() - if !result.Complete { +func (v *verifyingReader) verify() { + if v.verified { return } + v.verified = true - if len(r.checks.contentHash) > 0 { - if err := result.Verify(r.checks.contentHash); err != nil { - r.mismatched = true - r.onMismatch("content_hash: " + err.Error()) + if v.sha256 != nil { + got := hex.EncodeToString(v.sha256.Sum(nil)) + if subtle.ConstantTimeCompare([]byte(got), []byte(v.wantSHA256)) != 1 { + v.onMismatch(fmt.Sprintf("content_hash mismatch: stored=%s computed=%s", v.wantSHA256, got)) } } - if len(r.checks.native) > 0 { - if err := result.Verify(r.checks.native); err != nil { - r.mismatched = true - r.onMismatch("integrity: " + err.Error()) - } - } - if r.checks.hasChecksum { - if err := result.VerifyDigest(r.checks.checksum); err != nil { - r.mismatched = true - r.onMismatch("integrity: " + err.Error()) + if v.sri != nil { + got := v.sri.Sum(nil) + if subtle.ConstantTimeCompare(got, v.wantSRI) != 1 { + v.onMismatch(fmt.Sprintf("integrity mismatch: %s expected=%s computed=%s", + v.sriAlgo, + base64.StdEncoding.EncodeToString(v.wantSRI), + base64.StdEncoding.EncodeToString(got))) } } } - -// sha1Checksum returns a registry's hexadecimal SHA-1 shasum as the -// "sha1-" checksum newIntegrityChecks accepts, or "" when it is empty or -// not a SHA-1 digest. -func sha1Checksum(shasum string) string { - if shasum == "" { - return "" - } - digest, err := integrity.ParseHex(integrity.SHA1, shasum) - if err != nil { - return "" - } - return digest.Checksum() -} diff --git a/internal/handler/integrity_test.go b/internal/handler/integrity_test.go index 84ce34d..93c448c 100644 --- a/internal/handler/integrity_test.go +++ b/internal/handler/integrity_test.go @@ -1,22 +1,13 @@ package handler import ( - "bytes" "crypto/sha256" "crypto/sha512" "encoding/base64" "encoding/hex" - "errors" - "fmt" "io" - "net/http" - "net/http/httptest" - "slices" "strings" "testing" - "testing/iotest" - - "github.com/git-pkgs/registries/fetch" ) func sha256Hex(data string) string { @@ -24,100 +15,42 @@ func sha256Hex(data string) string { return hex.EncodeToString(sum[:]) } -func sha256SRI(data string) string { - sum := sha256.Sum256([]byte(data)) - return "sha256-" + base64.StdEncoding.EncodeToString(sum[:]) -} - -func sha384SRI(data string) string { - sum := sha512.Sum384([]byte(data)) - return "sha384-" + base64.StdEncoding.EncodeToString(sum[:]) -} - func sha512SRI(data string) string { sum := sha512.Sum512([]byte(data)) return "sha512-" + base64.StdEncoding.EncodeToString(sum[:]) } -func wrapIntegrityReader(t *testing.T, source io.ReadCloser, contentHash, native string, onMismatch func(string)) io.ReadCloser { - t.Helper() - checks, err := newIntegrityChecks(contentHash, native) - if err != nil { - t.Fatalf("newIntegrityChecks: %v", err) - } - reader, err := checks.wrap(source, onMismatch) - if err != nil { - t.Fatalf("wrap: %v", err) - } - return reader -} - -func TestNewIntegrityChecksCollectsAlgorithms(t *testing.T) { - checks, err := newIntegrityChecks( - sha256Hex("hello"), - strings.Join([]string{sha256SRI("first"), sha512SRI("second"), sha384SRI("third"), sha512SRI("alternative")}, " "), - ) - if err != nil { - t.Fatal(err) - } - if len(checks.algorithms) != 5 { - t.Fatalf("algorithms = %v, want 5 entries", checks.algorithms) - } - if len(checks.native) != 4 { - t.Errorf("native digests = %d, want 4", len(checks.native)) - } -} - -func TestNewIntegrityChecksRejectsMalformedMetadata(t *testing.T) { +func TestParseSRI(t *testing.T) { tests := []struct { - name string - contentHash string - native string + name string + input string + algo string + ok bool }{ - {name: "short content hash", contentHash: "abc123"}, - {name: "non-hex content hash", contentHash: strings.Repeat("z", sha256.Size*2)}, - {name: "missing SRI separator", native: "sha512"}, - {name: "malformed SRI base64", native: "sha512-not!base64"}, - {name: "wrong SRI length", native: "sha512-" + base64.StdEncoding.EncodeToString([]byte("short"))}, - {name: "unsupported SRI algorithm", native: "md5-1B2M2Y8AsgTpgAmY7PhCfg=="}, - {name: "invalid SRI alternative", native: sha512SRI("valid") + " sha384-nope"}, + {"sha512", sha512SRI("hello"), "sha512", true}, + {"sha256", "sha256-" + base64.StdEncoding.EncodeToString([]byte("0123456789012345678901234567890123456789")), "sha256", true}, + {"empty", "", "", false}, + {"no dash", "sha512abc", "", false}, + {"bad base64", "sha512-not!base64", "", false}, + {"unsupported algo", "md5-" + base64.StdEncoding.EncodeToString([]byte("x")), "", false}, + {"multi hash takes first", sha512SRI("a") + " " + sha512SRI("b"), "sha512", true}, + {"whitespace", " " + sha512SRI("x") + " ", "sha512", true}, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if _, err := newIntegrityChecks(test.contentHash, test.native); err == nil { - t.Fatal("newIntegrityChecks returned nil error") - } - }) - } -} -// TestIntegrityChecksVerifyRegistryChecksum checks native metadata given as -// an algorithm-prefixed hex checksum, the form recorded for a registry's SHA-1 -// shasum, which is not SRI. -func TestIntegrityChecksVerifyRegistryChecksum(t *testing.T) { - const data = "hello world" - checksum := "sha1-" + sha1Hex(data) - checks, err := newIntegrityChecks(sha256Hex(data), checksum) - if err != nil { - t.Fatalf("newIntegrityChecks(%q): %v", checksum, err) - } - for _, tt := range []struct { - name, body string - wantMismatch bool - }{ - {"matching", data, false}, - {"mismatched", "other bytes", true}, - } { + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var reasons []string - reader, err := checks.wrapFailOnMismatch(io.NopCloser(strings.NewReader(tt.body)), func(reason string) { reasons = append(reasons, reason) }) - if err != nil { - t.Fatal(err) + algo, digest, ok := parseSRI(tt.input) + if ok != tt.ok { + t.Fatalf("ok = %v, want %v", ok, tt.ok) } - _, _ = io.ReadAll(reader) - gotNative := slices.ContainsFunc(reasons, func(r string) bool { return strings.HasPrefix(r, "integrity: ") }) - if gotNative != tt.wantMismatch { - t.Errorf("checksum mismatch reported = %v, want %v (reasons %q)", gotNative, tt.wantMismatch, reasons) + if !tt.ok { + return + } + if algo != tt.algo { + t.Errorf("algo = %q, want %q", algo, tt.algo) + } + if len(digest) == 0 { + t.Error("digest is empty") } }) } @@ -134,417 +67,70 @@ func TestVerifyingReader(t *testing.T) { sri string wantCalls int }{ - {name: "both match", hash: goodSHA, sri: goodSRI}, - {name: "SHA-256 only match", hash: goodSHA}, - {name: "SRI only match", sri: goodSRI}, - {name: "SHA-256 mismatch", hash: sha256Hex("other"), wantCalls: 1}, - {name: "SRI mismatch", sri: sha512SRI("other"), wantCalls: 1}, - {name: "both mismatch", hash: sha256Hex("other"), sri: sha512SRI("other"), wantCalls: 2}, - {name: "no checks"}, + {"both match", goodSHA, goodSRI, 0}, + {"sha256 only match", goodSHA, "", 0}, + {"sri only match", "", goodSRI, 0}, + {"sha256 mismatch", sha256Hex("other"), "", 1}, + {"sri mismatch", "", sha512SRI("other"), 1}, + {"both mismatch", sha256Hex("other"), sha512SRI("other"), 2}, + {"no checks", "", "", 0}, + {"unparseable sri ignored", goodSHA, "garbage", 0}, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { var calls []string - reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader(data)), test.hash, test.sri, + r := newVerifyingReader(io.NopCloser(strings.NewReader(data)), tt.hash, tt.sri, func(reason string) { calls = append(calls, reason) }) - got, err := io.ReadAll(reader) + got, err := io.ReadAll(r) if err != nil { t.Fatalf("ReadAll: %v", err) } if string(got) != data { t.Errorf("data corrupted: got %q", got) } - if err := reader.Close(); err != nil { + if err := r.Close(); err != nil { t.Fatalf("Close: %v", err) } - if len(calls) != test.wantCalls { - t.Errorf("onMismatch called %d times, want %d: %v", len(calls), test.wantCalls, calls) + + if len(calls) != tt.wantCalls { + t.Errorf("onMismatch called %d times, want %d: %v", len(calls), tt.wantCalls, calls) } }) } } -func TestVerifyingReaderPreservesSeekCapability(t *testing.T) { - const data = "hello world" - t.Run("ordinary reads stay verified", func(t *testing.T) { - source := &seekableCloseTrackingReader{Reader: bytes.NewReader([]byte(data))} - var calls int - reader := wrapIntegrityReader(t, source, sha256Hex("different"), "", func(string) { calls++ }) - - got, err := io.ReadAll(reader) - if err != nil { - t.Fatalf("ReadAll: %v", err) - } - if string(got) != data { - t.Errorf("data = %q, want %q", got, data) - } - if calls != 1 { - t.Errorf("onMismatch called %d times, want 1", calls) - } - if err := reader.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - if source.closeCount != 1 { - t.Errorf("source closed %d times, want 1", source.closeCount) - } - }) - - t.Run("seek bypasses whole-object verification", func(t *testing.T) { - source := &seekableCloseTrackingReader{Reader: bytes.NewReader([]byte(data))} - var calls int - reader := wrapIntegrityReader(t, source, sha256Hex("different"), "", func(string) { calls++ }) - seeker, ok := reader.(io.Seeker) - if !ok { - t.Fatal("seekable source did not retain io.Seeker") - } - if _, err := seeker.Seek(6, io.SeekStart); err != nil { - t.Fatalf("Seek: %v", err) - } - - got, err := io.ReadAll(reader) - if err != nil { - t.Fatalf("ReadAll: %v", err) - } - if string(got) != "world" { - t.Errorf("data after seek = %q, want %q", got, "world") - } - if calls != 0 { - t.Errorf("onMismatch called %d times for a partial read, want 0", calls) - } - if err := reader.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - if source.closeCount != 1 { - t.Errorf("source closed %d times, want 1", source.closeCount) - } - }) -} - -type seekableCloseTrackingReader struct { - *bytes.Reader - closeCount int -} - -func (r *seekableCloseTrackingReader) Close() error { - r.closeCount++ - return nil -} - -func TestVerifyingReaderUsesStrongestNativeAlgorithm(t *testing.T) { - const data = "artifact" - tests := []struct { - name string - native string - wantCalls int - }{ - { - name: "weaker match does not override stronger mismatch", - native: sha256SRI(data) + " " + sha512SRI("other"), - wantCalls: 1, - }, - { - name: "stronger match ignores weaker mismatch", - native: sha256SRI("other") + " " + sha512SRI(data), - }, - { - name: "same algorithm alternative matches", - native: sha512SRI("other") + " " + sha512SRI(data), - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - var calls int - reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader(data)), "", test.native, func(string) { calls++ }) - if _, err := io.Copy(io.Discard, reader); err != nil { - t.Fatal(err) - } - if calls != test.wantCalls { - t.Errorf("onMismatch called %d times, want %d", calls, test.wantCalls) - } - }) - } -} - -func TestVerifyingReaderMismatchMessages(t *testing.T) { - const data = "actual" - wantHash := sha256Hex("expected") - wantSRI := sha512SRI("expected") - var reasons []string - reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader(data)), wantHash, wantSRI, - func(reason string) { reasons = append(reasons, reason) }) - if _, err := io.Copy(io.Discard, reader); err != nil { - t.Fatal(err) - } - if len(reasons) != 2 { - t.Fatalf("reasons = %v, want two", reasons) - } - wantContentReason := "content_hash: integrity mismatch: expected " + sha256SRI("expected") + ", calculated " + sha256SRI(data) - if reasons[0] != wantContentReason { - t.Errorf("content reason = %q, want %q", reasons[0], wantContentReason) - } - wantNativeReason := "integrity: integrity mismatch: expected " + wantSRI + ", calculated " + sha512SRI(data) - if reasons[1] != wantNativeReason { - t.Errorf("native reason = %q, want %q", reasons[1], wantNativeReason) - } -} - func TestVerifyingReaderPassthrough(t *testing.T) { - source := io.NopCloser(strings.NewReader("x")) - reader := wrapIntegrityReader(t, source, "", "", func(string) { t.Fatal("should not be called") }) - if reader != source { - t.Error("expected passthrough when no hashes were provided") + src := io.NopCloser(strings.NewReader("x")) + r := newVerifyingReader(src, "", "", func(string) { t.Fatal("should not be called") }) + if r != src { + t.Error("expected passthrough when no hashes provided") } } -type closeTrackingReader struct { - io.Reader - closed bool -} - -func (r *closeTrackingReader) Close() error { - r.closed = true - return nil -} - func TestVerifyingReaderPartialRead(t *testing.T) { - source := &closeTrackingReader{Reader: strings.NewReader("hello world")} var calls int - reader := wrapIntegrityReader(t, source, sha256Hex("other"), "", func(string) { calls++ }) + r := newVerifyingReader(io.NopCloser(strings.NewReader("hello world")), + sha256Hex("hello world"), "", func(string) { calls++ }) - buffer := make([]byte, 5) - _, _ = reader.Read(buffer) - _ = reader.Close() + buf := make([]byte, 5) + _, _ = r.Read(buf) + _ = r.Close() if calls != 0 { t.Errorf("onMismatch called %d times for partial read, want 0", calls) } - if !source.closed { - t.Error("Close was not forwarded to the source") - } -} - -func TestVerifyingReaderNonEOFError(t *testing.T) { - var calls int - reader := wrapIntegrityReader(t, io.NopCloser(errorFixtureReader{}), sha256Hex("data"), "", func(string) { calls++ }) - if _, err := io.ReadAll(reader); !errors.Is(err, errIntegrityReadFixture) { - t.Fatalf("ReadAll error = %v", err) - } - if calls != 0 { - t.Errorf("onMismatch called %d times after non-EOF error", calls) - } -} - -var errIntegrityReadFixture = errors.New("integrity read fixture") - -type errorFixtureReader struct{} - -func (errorFixtureReader) Read(p []byte) (int, error) { - return copy(p, "data"), errIntegrityReadFixture } func TestVerifyingReaderVerifyOnce(t *testing.T) { var calls int - reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader("x")), sha256Hex("y"), "", func(string) { calls++ }) - _, _ = io.ReadAll(reader) - _ = reader.Close() - _ = reader.Close() + r := newVerifyingReader(io.NopCloser(strings.NewReader("x")), sha256Hex("y"), "", + func(string) { calls++ }) + _, _ = io.ReadAll(r) + _ = r.Close() + _ = r.Close() if calls != 1 { t.Errorf("onMismatch called %d times, want 1", calls) } } - -// TestCorruptCachedArtifactAbortsResponse serves a cached artifact whose -// stored bytes no longer match its digest through a real server and client. -// The client must see a failed, short body rather than a complete response. -func TestCorruptCachedArtifactAbortsResponse(t *testing.T) { - proxy, db, store, _ := setupTestProxy(t) - content := strings.Repeat("cached content ", 10000) - seedPackage(t, db, store, "npm", "corrupt", "1.0.0", "corrupt-1.0.0.tgz", content) - seeded, err := db.GetArtifact("pkg:npm/corrupt@1.0.0", "corrupt-1.0.0.tgz") - if err != nil || seeded == nil || !seeded.StoragePath.Valid { - t.Fatalf("seeded record = %+v (err %v), want a storage path", seeded, err) - } - tampered := []byte(content) - tampered[len(tampered)-1] = '!' - store.files[seeded.StoragePath.String] = tampered - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - result, err := proxy.GetCachedArtifact(r.Context(), "npm", "corrupt", "1.0.0", "corrupt-1.0.0.tgz") - if err != nil || result == nil { - http.Error(w, "not cached", http.StatusInternalServerError) - return - } - ServeArtifactRequest(w, r, result) - })) - t.Cleanup(server.Close) - - resp, err := server.Client().Get(server.URL) - if err != nil { - t.Fatalf("GET: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) - } - if resp.ContentLength != int64(len(content)) { - t.Fatalf("Content-Length = %d, want %d", resp.ContentLength, len(content)) - } - body, err := io.ReadAll(resp.Body) - if err == nil { - t.Error("reading a corrupted cached artifact succeeded, want an error") - } - if int64(len(body)) >= resp.ContentLength { - t.Errorf("read %d bytes, want fewer than Content-Length %d", len(body), resp.ContentLength) - } - if !bytes.HasPrefix(tampered, body) { - t.Error("body is not a prefix of the stored bytes") - } - - artifact, err := db.GetArtifact("pkg:npm/corrupt@1.0.0", "corrupt-1.0.0.tgz") - if err != nil || artifact == nil || artifact.StoragePath.Valid { - t.Errorf("record = %+v (err %v), want it cleared", artifact, err) - } -} - -func TestFailOnMismatchHoldsBackFinalChunk(t *testing.T) { - data := strings.Repeat("0123456789", 1000) - for _, tt := range []struct { - name string - source func(string) io.Reader - }{ - {"single reads", func(s string) io.Reader { return strings.NewReader(s) }}, - {"one byte reads", func(s string) io.Reader { return iotest.OneByteReader(strings.NewReader(s)) }}, - {"data with EOF", func(s string) io.Reader { return iotest.DataErrReader(strings.NewReader(s)) }}, - } { - t.Run(tt.name, func(t *testing.T) { - checks, err := newIntegrityChecks(sha256Hex(data), "") - if err != nil { - t.Fatal(err) - } - - reader, err := checks.wrapFailOnMismatch(io.NopCloser(tt.source(data)), func(string) { - t.Error("onMismatch called for matching bytes") - }) - if err != nil { - t.Fatal(err) - } - got, err := io.ReadAll(reader) - if err != nil || string(got) != data { - t.Errorf("matching read = %d bytes, %v; want %d bytes, nil", len(got), err, len(data)) - } - - var calls int - tampered := data[:len(data)-1] + "!" - reader, err = checks.wrapFailOnMismatch(io.NopCloser(tt.source(tampered)), func(string) { calls++ }) - if err != nil { - t.Fatal(err) - } - got, err = io.ReadAll(reader) - if !errors.Is(err, ErrArtifactDigestMismatch) { - t.Errorf("mismatched read error = %v, want ErrArtifactDigestMismatch", err) - } - if len(got) >= len(tampered) || !strings.HasPrefix(tampered, string(got)) { - t.Errorf("mismatched read returned %d of %d bytes, want a strict prefix", len(got), len(tampered)) - } - if calls != 1 { - t.Errorf("onMismatch called %d times, want 1", calls) - } - }) - } -} - -// TestFetchWithDigestVerifiesEveryAlgorithm checks that a sha512 digest is -// verified before the artifact is cached, like a sha256 one, and that a digest -// whose algorithm can't be verified is refused without fetching. -func TestFetchWithDigestVerifiesEveryAlgorithm(t *testing.T) { - const content = "blob content" - sum512 := sha512.Sum512([]byte(content)) - tests := []struct { - name, digest string - wantErr error - }{ - {"sha256", "sha256:" + sha256Hex(content), nil}, - {"sha512", "sha512:" + hex.EncodeToString(sum512[:]), nil}, - {"mismatched sha256", "sha256:" + sha256Hex("other"), ErrArtifactDigestMismatch}, - {"mismatched sha512", "sha512:" + strings.Repeat("a", 128), ErrArtifactDigestMismatch}, - {"unsupported algorithm", "sha1:" + strings.Repeat("a", 40), ErrUnsupportedDigest}, - {"malformed sha512", "sha512:abc", ErrUnsupportedDigest}, - } - for _, stream := range []bool{false, true} { - for _, tt := range tests { - t.Run(fmt.Sprintf("%s/stream=%t", tt.name, stream), func(t *testing.T) { - checkFetchWithDigest(t, stream, content, tt.digest, tt.wantErr) - }) - } - } -} - -func checkFetchWithDigest(t *testing.T, stream bool, content, digest string, wantErr error) { - t.Helper() - proxy, _, _, fetcher := setupTestProxy(t) - proxy.StreamArtifacts = stream - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader(content))} - - result, err := proxy.GetOrFetchArtifactFromURLWithDigest(t.Context(), "oci", "library/demo", digest, digest, - "https://registry.example/v2/library/demo/blobs/"+digest, digest) - if err == nil { - var body []byte - body, err = io.ReadAll(result.Reader) - _ = result.Reader.Close() - if err == nil && string(body) != content { - t.Errorf("body = %q, want %q", body, content) - } - } - if !errors.Is(err, wantErr) { - t.Fatalf("error = %v, want %v", err, wantErr) - } - if errors.Is(wantErr, ErrUnsupportedDigest) && fetcher.fetchCalled { - t.Error("fetched an artifact whose digest can't be verified") - } - cached, err := proxy.GetCachedArtifact(t.Context(), "oci", "library/demo", digest, digest) - if err != nil { - t.Fatal(err) - } - if cached != nil { - _ = cached.Reader.Close() - } - if wantCached := wantErr == nil && !stream; (cached != nil) != wantCached { - t.Errorf("cached = %v, want %v", cached != nil, wantCached) - } -} - -// BenchmarkStoredObjectVerification compares verifying a cache hit with -// every recorded digest (SHA-256 content hash plus npm's SHA-512) against -// the content hash alone. -func BenchmarkStoredObjectVerification(b *testing.B) { - data := strings.Repeat("x", 8<<20) - checks, err := newIntegrityChecks(sha256Hex(data), sha512SRI(data)) - if err != nil { - b.Fatal(err) - } - for _, bench := range []struct { - name string - checks integrityChecks - }{ - {"all digests", checks}, - {"content hash", checks.forStoredObject()}, - } { - b.Run(bench.name, func(b *testing.B) { - b.SetBytes(int64(len(data))) - buf := make([]byte, 32<<10) - for b.Loop() { - reader, err := bench.checks.wrapFailOnMismatch(io.NopCloser(strings.NewReader(data)), func(reason string) { - b.Fatal(reason) - }) - if err != nil { - b.Fatal(err) - } - if _, err := io.CopyBuffer(io.Discard, reader, buf); err != nil { - b.Fatal(err) - } - } - }) - } -} diff --git a/internal/handler/julia.go b/internal/handler/julia.go index dbac313..08b1fdf 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -12,10 +12,8 @@ import ( "regexp" "strings" "sync" - "time" "github.com/BurntSushi/toml" - "github.com/git-pkgs/proxy/internal/retention" ) const ( @@ -23,11 +21,6 @@ const ( juliaGeneralRegistryUUID = "23338594-aafe-5451-b93e-139f81909106" juliaArtifactName = "_artifact" juliaRegistryName = "_registry" - - // juliaRefreshTimeout bounds a background reload of the name map. - juliaRefreshTimeout = 5 * time.Minute - // maxJuliaRegistryTomlSize bounds Registry.toml. General's is a few MB. - maxJuliaRegistryTomlSize = 64 << 20 ) var ( @@ -35,14 +28,6 @@ var ( juliaUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) ) -// Julia takes no package rules: its PURLs need a uuid qualifier the cache -// doesn't store. The julia rule and the default cover package tarballs, -// artifacts and registry snapshots alike; all are content-addressed, so an -// evicted one comes back unchanged. -func init() { - retention.Default.Register(retention.Spec{Key: "julia"}) -} - // JuliaHandler handles Julia Pkg server protocol requests. // // See https://pkgdocs.julialang.org/v1/registries/ and the PkgServer.jl @@ -68,13 +53,6 @@ func NewJuliaHandler(proxy *Proxy, _ string) *JuliaHandler { } } -// NewJuliaHandlerWithUpstream creates a Julia handler with a custom upstream. -func NewJuliaHandlerWithUpstream(proxy *Proxy, upstreamURL string) *JuliaHandler { - h := NewJuliaHandler(proxy, "") - h.upstreamURL = configuredUpstreamURL(upstreamURL, juliaUpstream) - return h -} - // Routes returns the HTTP handler for Julia requests. func (h *JuliaHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -107,18 +85,19 @@ func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) { return } - h.proxy.Logger.Debug("julia registry request", "uuid", uuid, "hash", hash) + h.proxy.Logger.Info("julia registry request", "uuid", uuid, "hash", hash) upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, juliaTarball(hash), upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, hash+".tar.gz", upstreamURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch registry") + h.proxy.Logger.Error("failed to get registry", "error", err) + http.Error(w, "failed to fetch registry", http.StatusBadGateway) return } - go h.refreshNamesFromRegistry(context.WithoutCancel(r.Context()), uuid, hash) + go h.refreshNamesFromRegistry(uuid, hash) - ServeArtifactRequest(w, r, result) + ServeArtifact(w, result) } // handlePackage serves an immutable package source tarball. @@ -137,69 +116,15 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("julia package request", "name", name, "uuid", uuid, "hash", hash) - if name == uuid { - if result := h.cachedPackage(r.Context(), hash); result != nil { - ServeArtifactRequest(w, r, result) - return - } - } - upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, juliaTarball(hash), upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, hash+".tar.gz", upstreamURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") + h.proxy.Logger.Error("failed to get package", "error", err) + http.Error(w, "failed to fetch package", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) -} - -// cachedPackage returns the package tarball with tree hash hash from the -// cache, under whatever name it was cached as, or nil. Without the name map, -// which needs the registry from upstream, a request can't be matched to the -// name its package was cached under, but the tarball is identified by its -// tree hash alone. The registry and artifact tarballs cached under the same -// file name are skipped. -func (h *JuliaHandler) cachedPackage(ctx context.Context, hash string) *CacheResult { - if h.proxy.DB == nil { - return nil - } - filename := juliaTarball(hash) - names, err := h.proxy.DB.CachedArtifactNames("julia", filename) - if err != nil { - h.proxy.Logger.Warn("looking up cached julia package", "hash", hash, "error", err) - return nil - } - for _, name := range names { - if name == juliaRegistryName || name == juliaArtifactName { - continue - } - result, err := h.proxy.GetCachedArtifact(ctx, "julia", name, hash, filename) - if err == nil && result != nil { - return result - } - } - return nil -} - -// JuliaPackageDownload returns the URL of a package's tarball on the Pkg -// server at upstreamURL, or the default one when empty, and the file name the -// Julia handler caches it under. -func JuliaPackageDownload(upstreamURL, uuid, treeHash string) (url, filename string, err error) { - if !validJuliaUUID(uuid) { - return "", "", fmt.Errorf("invalid julia package uuid %q", uuid) - } - if !juliaHexPattern.MatchString(treeHash) { - return "", "", fmt.Errorf("invalid julia tree hash %q", treeHash) - } - base := configuredUpstreamURL(upstreamURL, juliaUpstream) - return base + "/package/" + uuid + "/" + treeHash, juliaTarball(treeHash), nil -} - -// juliaTarball is the file name a registry, package or artifact tarball is -// cached under. -func juliaTarball(hash string) string { - return hash + ".tar.gz" + ServeArtifact(w, result) } // handleArtifact serves an immutable binary artifact tarball. Artifacts are @@ -211,16 +136,17 @@ func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { return } - h.proxy.Logger.Debug("julia artifact request", "hash", hash) + h.proxy.Logger.Info("julia artifact request", "hash", hash) upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, juliaTarball(hash), upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, hash+".tar.gz", upstreamURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch artifact") + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch artifact", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) + ServeArtifact(w, result) } // proxyUpstream forwards a request to the upstream Pkg server without caching. @@ -272,16 +198,24 @@ func (h *JuliaHandler) loadNamesFromUpstream(ctx context.Context) error { return h.loadRegistryTarball(ctx, juliaGeneralRegistryUUID, hash) } -// fetchGeneralRegistryHash reads /registries, through the metadata cache that -// also serves it to clients, and returns the current tree hash for the -// General registry. +// fetchGeneralRegistryHash reads /registries and returns the current tree hash +// for the General registry. func (h *JuliaHandler) fetchGeneralRegistryHash(ctx context.Context) (string, error) { - body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "julia", "registries", h.upstreamURL+"/registries", "*/*") + req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.upstreamURL+"/registries", nil) if err != nil { - return "", fmt.Errorf("reading /registries: %w", err) + return "", err + } + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("upstream /registries returned %d", resp.StatusCode) } - scanner := bufio.NewScanner(bytes.NewReader(body)) + scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { uuid, hash, ok := parseRegistryLine(scanner.Text()) if ok && uuid == juliaGeneralRegistryUUID { @@ -294,39 +228,21 @@ func (h *JuliaHandler) fetchGeneralRegistryHash(ctx context.Context) (string, er return "", fmt.Errorf("general registry not listed in /registries") } -// refreshNamesFromRegistry reloads the UUID→name map after a General registry -// tarball has been served. Only the hash /registries currently lists replaces -// the map, so requests for older snapshots neither roll it back nor trigger a -// download each. One reload runs at a time; a request that finds one running -// skips its own. Errors are logged but do not affect the response. -func (h *JuliaHandler) refreshNamesFromRegistry(ctx context.Context, uuid, hash string) { - if uuid != juliaGeneralRegistryUUID || h.currentNamesHash() == hash { +// refreshNamesFromRegistry reloads the UUID→name map from a registry tarball +// that has just been cached. Errors are logged but do not affect the response. +func (h *JuliaHandler) refreshNamesFromRegistry(uuid, hash string) { + if uuid != juliaGeneralRegistryUUID { return } - if !h.loadMu.TryLock() { - return - } - defer h.loadMu.Unlock() - - ctx, cancel := context.WithTimeout(ctx, juliaRefreshTimeout) - defer cancel() - advertised, err := h.fetchGeneralRegistryHash(ctx) - if err != nil { - h.proxy.Logger.Warn("failed to refresh julia name map", "error", err) - return - } - if advertised != hash || h.currentNamesHash() == hash { - return - } - if err := h.loadRegistryTarball(ctx, uuid, hash); err != nil { - h.proxy.Logger.Warn("failed to refresh julia name map", "error", err) - } -} - -func (h *JuliaHandler) currentNamesHash() string { h.mu.RLock() - defer h.mu.RUnlock() - return h.namesHash + current := h.namesHash + h.mu.RUnlock() + if current == hash { + return + } + if err := h.loadRegistryTarball(context.Background(), uuid, hash); err != nil { + h.proxy.Logger.Warn("failed to refresh julia name map", "error", err) + } } // loadRegistryTarball downloads a registry tarball and replaces the name map @@ -383,13 +299,10 @@ func extractRegistryNames(r io.Reader) (map[string]string, error) { continue } - data, err := io.ReadAll(io.LimitReader(tr, maxJuliaRegistryTomlSize+1)) + data, err := io.ReadAll(tr) if err != nil { return nil, err } - if len(data) > maxJuliaRegistryTomlSize { - return nil, fmt.Errorf("registry file Registry.toml exceeds %d bytes", maxJuliaRegistryTomlSize) - } return parseRegistryToml(data) } } diff --git a/internal/handler/julia_refresh_test.go b/internal/handler/julia_refresh_test.go deleted file mode 100644 index 4ee080b..0000000 --- a/internal/handler/julia_refresh_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package handler - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "context" - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - "time" -) - -func juliaRegistryTarball(t *testing.T, packageName string) []byte { - t.Helper() - body := "name = \"General\"\n[packages]\n295af30f-e4ad-537b-8983-00126c2a3abe = { name = \"" + packageName + "\", path = \"R/X\" }\n" - var buf bytes.Buffer - gw := gzip.NewWriter(&buf) - tw := tar.NewWriter(gw) - if err := tw.WriteHeader(&tar.Header{Name: "Registry.toml", Mode: 0o644, Size: int64(len(body))}); err != nil { - t.Fatal(err) - } - _, _ = tw.Write([]byte(body)) - _ = tw.Close() - _ = gw.Close() - return buf.Bytes() -} - -type fakeJuliaUpstream struct { - mu sync.Mutex - advertised string - tarballs map[string][]byte - downloads map[string]int -} - -func (u *fakeJuliaUpstream) ServeHTTP(w http.ResponseWriter, r *http.Request) { - u.mu.Lock() - defer u.mu.Unlock() - if r.URL.Path == "/registries" { - _, _ = w.Write([]byte("/registry/" + juliaGeneralRegistryUUID + "/" + u.advertised + "\n")) - return - } - hash := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] - body, ok := u.tarballs[hash] - if !ok { - http.NotFound(w, r) - return - } - u.downloads[hash]++ - _, _ = w.Write(body) -} - -func (u *fakeJuliaUpstream) count(hash string) int { - u.mu.Lock() - defer u.mu.Unlock() - return u.downloads[hash] -} - -func TestJuliaRefreshOnlyLoadsAdvertisedRegistry(t *testing.T) { - oldHash, newHash := strings.Repeat("a", 40), strings.Repeat("b", 40) - up := &fakeJuliaUpstream{ - advertised: newHash, - tarballs: map[string][]byte{ - oldHash: juliaRegistryTarball(t, "OldName"), - newHash: juliaRegistryTarball(t, "NewName"), - }, - downloads: map[string]int{}, - } - server := httptest.NewServer(up) - defer server.Close() - - proxy, _, _, _ := setupTestProxy(t) - h := NewJuliaHandlerWithUpstream(proxy, server.URL) - ctx := context.Background() - - // An older snapshot neither replaces the map nor is downloaded for it. - for range 5 { - h.refreshNamesFromRegistry(ctx, juliaGeneralRegistryUUID, oldHash) - } - if n := up.count(oldHash); n != 0 { - t.Fatalf("old registry downloaded %d times for the name map", n) - } - - h.refreshNamesFromRegistry(ctx, juliaGeneralRegistryUUID, newHash) - if got := h.resolveName("295af30f-e4ad-537b-8983-00126c2a3abe"); got != "NewName" { - t.Fatalf("name = %q, want NewName", got) - } - - // The loaded hash is not fetched again, and an old one doesn't roll back. - h.refreshNamesFromRegistry(ctx, juliaGeneralRegistryUUID, newHash) - h.refreshNamesFromRegistry(ctx, juliaGeneralRegistryUUID, oldHash) - if n := up.count(newHash); n != 1 { - t.Errorf("current registry downloaded %d times, want 1", n) - } - if got := h.resolveName("295af30f-e4ad-537b-8983-00126c2a3abe"); got != "NewName" { - t.Errorf("name rolled back to %q", got) - } -} - -func TestJuliaRefreshSkipsWhileLoading(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - h := NewJuliaHandlerWithUpstream(proxy, "http://127.0.0.1:1") - h.loadMu.Lock() - done := make(chan struct{}) - go func() { - h.refreshNamesFromRegistry(context.Background(), juliaGeneralRegistryUUID, strings.Repeat("c", 40)) - close(done) - }() - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("refresh waited for a running load instead of skipping") - } - h.loadMu.Unlock() -} - -func TestJuliaExtractRegistryNamesLimitsRegistryToml(t *testing.T) { - var buf bytes.Buffer - gw := gzip.NewWriter(&buf) - tw := tar.NewWriter(gw) - size := int64(maxJuliaRegistryTomlSize + 1) - if err := tw.WriteHeader(&tar.Header{Name: "Registry.toml", Mode: 0o644, Size: size}); err != nil { - t.Fatal(err) - } - if _, err := tw.Write(bytes.Repeat([]byte("#"), int(size))); err != nil { - t.Fatal(err) - } - _ = tw.Close() - _ = gw.Close() - if _, err := extractRegistryNames(&buf); err == nil { - t.Fatal("oversized Registry.toml was accepted") - } -} diff --git a/internal/handler/maven.go b/internal/handler/maven.go index e841d08..c423645 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -1,17 +1,11 @@ package handler import ( - "context" - "crypto/sha256" - "encoding/hex" "errors" "fmt" "net/http" "path" "strings" - - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/purl" ) const ( @@ -20,62 +14,12 @@ const ( minMavenParts = 4 // group path segments + artifact + version + filename ) -// Maven files are stored under pkg:maven/{group}/{artifact}, built from the -// group:artifact name, which is what the default key mapping produces from -// a package rule. Names keep their case, as Maven coordinates do. -func init() { - retention.Default.Register(retention.Spec{Key: "maven", CanonicalPackage: mavenRetentionKey}) -} - -// mavenRetentionKey maps a package rule to the stored package PURL. A key -// without a group, or with the group written as a path (com/google/guava) -// instead of dotted, parses fine but can never match a stored package, so -// it is refused rather than accepted as a rule that does nothing. -func mavenRetentionKey(p *purl.PURL) (string, bool) { - if p == nil || p.Namespace == "" || strings.Contains(p.Namespace, "/") { - return "", false - } - return retention.DefaultCanonical("maven")(p) -} - -// MavenHandler handles Maven repository protocol requests. The default -// repository is served at /maven/ and falls back to the Gradle Plugin Portal; -// each named repository is served at /maven/{name}/ from its own upstream -// alone. +// MavenHandler handles Maven repository protocol requests. type MavenHandler struct { proxy *Proxy upstreamURL string pluginPortalUpstreamURL string proxyURL string - repositories map[string]mavenSource -} - -// mavenSource is the repository a request is served from. -type mavenSource struct { - // name is the repository name, or "" for the default repository. - name string - upstreamURL string - // fallbackURL is tried when upstreamURL doesn't have a file. Only the - // default repository has one, the Gradle Plugin Portal. - fallbackURL string - // cachePrefix is prepended to cached filenames: empty for the default - // repository, {name}/{hash of upstreamURL}/ for a named one. - cachePrefix string -} - -// WithRepositories adds named repositories, served at /maven/{name}/. -func (h *MavenHandler) WithRepositories(repositories map[string]string) *MavenHandler { - h.repositories = make(map[string]mavenSource, len(repositories)) - for name, upstreamURL := range repositories { - upstreamURL = strings.TrimSuffix(upstreamURL, "/") - digest := sha256.Sum256([]byte(upstreamURL)) - h.repositories[name] = mavenSource{ - name: name, - upstreamURL: upstreamURL, - cachePrefix: name + "/" + hex.EncodeToString(digest[:mavenUpstreamHashBytes]) + "/", - } - } - return h } // NewMavenHandler creates a new Maven repository handler. @@ -103,7 +47,7 @@ func (h *MavenHandler) Routes() http.Handler { // e.g., /com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar mux.HandleFunc("GET /", h.handleRequest) - return rejectQueryChars(mux) + return mux } // handleRequest routes Maven requests based on the path. @@ -114,166 +58,93 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) { return } - source := mavenSource{upstreamURL: h.upstreamURL, fallbackURL: h.pluginPortalUpstreamURL} - if name, rest, ok := strings.Cut(urlPath, "/"); ok { - if named, found := h.repositories[name]; found { - source = named - urlPath = rest - } - } else if _, found := h.repositories[urlPath]; found { - http.NotFound(w, r) - return - } - if urlPath == "" { - http.NotFound(w, r) - return - } - + // Check if this is an artifact file or metadata filename := path.Base(urlPath) - if h.isVersionListingFile(filename) { - h.handleMetadata(w, r, source, urlPath) + if h.isMetadataFile(filename) { + h.handleMetadata(w, r, urlPath) return } - if h.isArtifactFile(filename) || h.isChecksumFile(filename) || h.isCoordinateFile(urlPath) { - h.handleDownload(w, r, source, urlPath) + if h.isArtifactFile(filename) { + // Cache artifact files + h.handleDownload(w, r, urlPath) return } - // Proxy everything else (directory listings, catalogs etc.) - h.proxyUpstream(w, r, source, urlPath) + // Proxy everything else (directory listings, checksums, etc.) + h.proxyUpstream(w, r) } -// metadataCacheKey is the metadata cache key of urlPath. The default -// repository's keys are hashed from its upstream URLs and the path. A named -// repository's keys are hashed from its name, upstream URL and the path, so -// repositories never share entries and repointing one drops what it cached. -func (s mavenSource) metadataCacheKey(urlPath string) string { - if s.name == "" { - return hashedCacheKey(s.upstreamURL, s.fallbackURL, urlPath) - } - digest := sha256.Sum256([]byte(s.name + "\x00" + s.upstreamURL + "\x00" + urlPath)) - return "repo_" + hex.EncodeToString(digest[:]) -} +func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, urlPath string) { + cacheKey := strings.ReplaceAll(urlPath, "/", "_") + upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, urlPath) -// cacheFilename is the name a file is cached under. A named repository's -// files are prefixed with its name and a hash of its upstream URL, so the -// same coordinates in two repositories, which may hold different bytes, are -// cached apart, and repointing a repository stops serving what the old -// upstream sent. -func (s mavenSource) cacheFilename(filename string) string { - return s.cachePrefix + filename -} - -// mavenUpstreamHashBytes is how much of the upstream URL's hash a named -// repository's cache filenames carry: enough to tell its upstreams apart. -const mavenUpstreamHashBytes = 8 - -func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { - if h.cooldownEnabled() && h.serveCooldownMetadata(w, r, source, urlPath) { - return - } - - body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, urlPath) + body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, upstreamURL, "*/*") if err != nil { - h.writeMetadataError(w, err) + if errors.Is(err, ErrUpstreamNotFound) { + pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath) + h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal", + "path", urlPath) + body, contentType, err = h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, pluginPortalURL, "*/*") + } + } + if err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + h.proxy.Logger.Error("metadata fetch failed", "error", err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType) } -// fetchMetadata fetches a metadata file from the source's upstream, or from -// its fallback when the upstream doesn't have it, and returns it with the -// key it is cached under. -func (h *MavenHandler) fetchMetadata(ctx context.Context, source mavenSource, urlPath string) (body []byte, contentType, cacheKey string, err error) { - cacheKey = source.metadataCacheKey(urlPath) - upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) - - body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, upstreamURL, "*/*") - if err != nil && source.fallbackURL != "" { - if errors.Is(err, ErrUpstreamNotFound) { - pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) - h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal", - "path", urlPath) - body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, pluginPortalURL, "*/*") - } - } - return body, contentType, cacheKey, err -} - -func (h *MavenHandler) writeMetadataError(w http.ResponseWriter, err error) { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("metadata fetch failed", "error", err) - http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) -} - // handleDownload serves an artifact file, fetching and caching from upstream if needed. -func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { +func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, urlPath string) { // Parse Maven path: group/artifact/version/filename // e.g., com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar group, artifact, version, filename := h.parsePath(urlPath) if artifact == "" { - if strings.Count(urlPath, "/") >= minMavenParts-1 { - // A dot or colon in a group directory, or a colon in the - // artifact, which the Maven layout never has. Proxied, the - // file would skip the denylist, scan and cooldown. - http.NotFound(w, r) - return - } - h.proxyUpstream(w, r, source, urlPath) - return - } - - // The .pom of a version whose time isn't known yet carries it in its - // own Last-Modified, so it is checked once fetched, without a request - // of its own. Maven fetches the .pom before a version's other files. - ctx, checkAfter := h.cooldownFromDownload(r.Context(), source, urlPath, version) - if !checkAfter && h.checkCooldown(w, r, source, urlPath, version) { + h.proxyUpstream(w, r) return } // Maven uses group:artifact as the package name name := fmt.Sprintf("%s:%s", group, artifact) - h.proxy.Logger.Debug("maven download request", "repository", source.name, + h.proxy.Logger.Info("maven download request", "group", group, "artifact", artifact, "version", version, "filename", filename) - upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) - cacheFilename := source.cacheFilename(filename) + upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, urlPath) - result, err := h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, upstreamURL) - if err != nil && source.fallbackURL != "" { + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, upstreamURL) + if err != nil { if errors.Is(err, ErrUpstreamNotFound) { - pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) + pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath) h.proxy.Logger.Info("maven artifact not found in primary upstream, trying Gradle Plugin Portal", "group", group, "artifact", artifact, "version", version, "filename", filename) - result, err = h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, pluginPortalURL) + result, err = h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, pluginPortalURL) } } if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch artifact") - return - } - if checkAfter && h.checkCooldown(w, r, source, urlPath, version) { - _ = result.Reader.Close() + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch artifact", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) + ServeArtifact(w, result) } // parsePath extracts Maven coordinates from a URL path. // e.g., "com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar" // -> ("com.google.guava", "guava", "32.1.3-jre", "guava-32.1.3-jre.jar") -// A path that isn't in the Maven layout returns empty strings. func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, filename string) { parts := strings.Split(urlPath, "/") if len(parts) < minMavenParts { @@ -284,18 +155,6 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file version = parts[len(parts)-2] artifact = parts[len(parts)-3] groupParts := parts[:len(parts)-3] - // The Maven layout splits a group at its dots, so a group directory - // never holds a dot or is empty. Such a path would spell the same group - // as another and share its cache entries, so it is not parsed. Nor is a - // colon in the group or artifact, which are cached as group:artifact. - for _, part := range groupParts { - if part == "" || strings.ContainsAny(part, ".:") { - return "", "", "", "" - } - } - if strings.Contains(artifact, ":") { - return "", "", "", "" - } group = strings.Join(groupParts, ".") return group, artifact, version, filename @@ -303,12 +162,8 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file // isArtifactFile returns true if the filename looks like a Maven artifact. func (h *MavenHandler) isArtifactFile(filename string) bool { - // Common artifact extensions, and the archives distributions such as - // apache-maven-{version}-bin.tar.gz are published as. - extensions := []string{ - ".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module", - ".zip", ".tar.gz", ".tgz", ".tar.bz2", ".tar.xz", - } + // Common artifact extensions + extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module"} for _, ext := range extensions { if strings.HasSuffix(filename, ext) { return true @@ -317,59 +172,17 @@ func (h *MavenHandler) isArtifactFile(filename string) bool { return false } -// isCoordinateFile reports whether urlPath is a file named after the -// artifact and version of its directory, {artifact}-{version}..., as Maven -// names every file it publishes, whatever the extension. A SNAPSHOT's -// files carry a timestamp in place of "SNAPSHOT". -// -// The group isn't checked, so a file under a group directory outside the -// Maven layout still goes to handleDownload, which answers 404. -func (h *MavenHandler) isCoordinateFile(urlPath string) bool { - parts := strings.Split(urlPath, "/") - if len(parts) < minMavenParts { - return false - } - filename, version, artifact := parts[len(parts)-1], parts[len(parts)-2], parts[len(parts)-3] - if artifact == "" || version == "" { - return false - } - return strings.HasPrefix(filename, artifact+"-"+strings.TrimSuffix(version, "SNAPSHOT")) +// isMetadataFile returns true if the filename is Maven metadata. +func (h *MavenHandler) isMetadataFile(filename string) bool { + return filename == "maven-metadata.xml" || + strings.HasSuffix(filename, ".sha1") || + strings.HasSuffix(filename, ".sha256") || + strings.HasSuffix(filename, ".sha512") || + strings.HasSuffix(filename, ".md5") || + strings.HasSuffix(filename, ".asc") } -// isVersionListingFile matches maven-metadata.xml and its checksums. The -// listing changes with every release, so it is cached for the metadata TTL -// rather than kept like an artifact. -func (h *MavenHandler) isVersionListingFile(filename string) bool { - return strings.HasPrefix(filename, "maven-metadata.xml") -} - -// isChecksumFile matches the checksum and signature files published beside -// an artifact. They are as immutable as the artifact, so they are cached -// with it rather than refetched on every request. -func (h *MavenHandler) isChecksumFile(filename string) bool { - for _, ext := range []string{".sha1", ".sha256", ".sha512", ".md5", ".asc"} { - if strings.HasSuffix(filename, ext) { - return true - } - } - return false -} - -// mavenForwardHeaders lets clients resume artifacts proxied without caching. -var mavenForwardHeaders = []string{"Range", "If-Range"} - -// proxyUpstream forwards a request to the source's upstream without caching. -func (h *MavenHandler) proxyUpstream(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { - if source.fallbackURL == "" { - h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, mavenForwardHeaders) - return - } - // As for cached files, a file the primary upstream lacks is looked for - // on the Gradle Plugin Portal. - resp, err := h.proxy.upstreamRequest(r, source.upstreamURL+"/"+urlPath, mavenForwardHeaders) - if err == nil && resp.StatusCode == http.StatusNotFound { - _ = resp.Body.Close() - resp, err = h.proxy.upstreamRequest(r, source.fallbackURL+"/"+urlPath, mavenForwardHeaders) - } - h.proxy.relayUpstream(w, r, resp, err) +// proxyUpstream forwards a request to Maven Central without caching. +func (h *MavenHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { + h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, nil) } diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go deleted file mode 100644 index 4da839e..0000000 --- a/internal/handler/maven_cooldown.go +++ /dev/null @@ -1,475 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "crypto/md5" - "crypto/sha1" - "crypto/sha256" - "crypto/sha512" - "encoding/hex" - "encoding/xml" - "errors" - "fmt" - "hash" - "io" - "net/http" - "net/url" - "path" - "slices" - "strings" - "time" -) - -// Maven cooldown measures a version by the Last-Modified header of its .pom. -// On Maven Central that is the time the version was deployed. The header is -// read once per version and stored in the versions table, since release -// files don't change. -// -// maven-metadata.xml has a single for the whole artifact, so -// the proxy only looks at versions that can still be in cooldown: none when -// is older than the window, and otherwise the versions from -// the end of the list back to the first one that is old enough. Maven -// Central blocks clients that send bursts of requests, so the proxy doesn't -// read every version's time. -// -// SNAPSHOT versions are left alone: their files change after publishing. - -const errMavenCooldown = "version is in cooldown" - -// mavenTimeUserAgent is the User-Agent of publish-time requests, the one -// the artifact fetcher sends. -const mavenTimeUserAgent = "git-pkgs-proxy/1.0" - -const ( - mavenMetadataFile = "maven-metadata.xml" - mavenSnapshot = "-SNAPSHOT" - // mavenLastUpdatedLayout is the UTC timestamp format of . - mavenLastUpdatedLayout = "20060102150405" -) - -// mavenChecksums are the checksum files served beside maven-metadata.xml. -// With cooldown on they are computed from the metadata as served. -var mavenChecksums = map[string]func() hash.Hash{ - ".md5": md5.New, - ".sha1": sha1.New, - ".sha256": sha256.New, - ".sha512": sha512.New, -} - -func (h *MavenHandler) cooldownEnabled() bool { - return h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() -} - -func mavenPackagePURL(group, artifact string) string { - return canonicalPackagePURL("maven", group+":"+artifact) -} - -// mavenCoordinates returns the group and artifact of an artifact directory -// such as com/google/guava/guava. -func mavenCoordinates(artifactDir string) (group, artifact string, ok bool) { - i := strings.LastIndex(artifactDir, "/") - if i <= 0 || i == len(artifactDir)-1 { - return "", "", false - } - return strings.ReplaceAll(artifactDir[:i], "/", "."), artifactDir[i+1:], true -} - -// usableMavenVersion reports whether version can be put in a URL path as -// one segment. Versions come from upstream metadata. -func usableMavenVersion(version string) bool { - return version != "" && version != "." && version != ".." && !strings.ContainsAny(version, "/\\?#%") -} - -// mavenTimeKey returns the versions-table PURLs a version's time is stored -// under. A named repository's keys carry its upstream as repository_url: -// the same coordinates there can have been published at another time than -// in the default repository. -func (h *MavenHandler) mavenTimeKey(source mavenSource, group, artifact, version string) (versionPURL, packagePURL string, ok bool) { - if h.proxy.DB == nil { - return "", "", false - } - name := group + ":" + artifact - versionPURL, packagePURL = canonicalVersionPURL("maven", name, version), canonicalPackagePURL("maven", name) - if source.name != "" { - qualifier := "?repository_url=" + url.QueryEscape(source.upstreamURL) - versionPURL += qualifier - packagePURL += qualifier - } - return versionPURL, packagePURL, true -} - -// publishTime returns a version's publish time from the versions table, or -// from the Last-Modified of its .pom, which is then stored. A version -// without a .pom, or whose .pom has no Last-Modified, gets the zero time and -// is let through, as in the other ecosystems. A failed request is an error, -// so a rate-limited upstream can't let a version through. -func (h *MavenHandler) publishTime(ctx context.Context, source mavenSource, artifactDir, version string) (time.Time, error) { - group, artifact, ok := mavenCoordinates(artifactDir) - if !ok || !usableMavenVersion(version) { - return time.Time{}, nil - } - versionPURL, packagePURL, store := h.mavenTimeKey(source, group, artifact, version) - if store { - if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return ver.PublishedAt.Time, nil - } - } - - pomPath := artifactDir + "/" + version + "/" + artifact + "-" + version + ".pom" - published, found, err := h.pomLastModified(ctx, source.upstreamURL+"/"+pomPath) - if err == nil && !found && source.fallbackURL != "" { - published, _, err = h.pomLastModified(ctx, source.fallbackURL+"/"+pomPath) - } - if err != nil { - return time.Time{}, err - } - if store && !published.IsZero() { - if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, published); err != nil { - h.proxy.Logger.Warn("cooldown: could not store maven publish time", - "group", group, "artifact", artifact, "version", version, "error", err) - } - } - return published, nil -} - -// cooldownFromDownload returns a context that stores the publish time of a -// version from the download of its .pom or Gradle .module, and whether the -// cooldown check is to run after the download. It does so only for those -// files of a version whose time is not stored yet; for anything else the -// check runs before. Both files are deployed with the version, and -// whichever a client fetches first gives its time. -func (h *MavenHandler) cooldownFromDownload(ctx context.Context, source mavenSource, urlPath, version string) (context.Context, bool) { - if !h.cooldownEnabled() || strings.HasSuffix(version, mavenSnapshot) { - return ctx, false - } - artifactDir := path.Dir(path.Dir(urlPath)) - group, artifact, ok := mavenCoordinates(artifactDir) - if base := artifact + "-" + version; !ok || !usableMavenVersion(version) || - (path.Base(urlPath) != base+".pom" && path.Base(urlPath) != base+".module") { - return ctx, false - } - versionPURL, packagePURL, store := h.mavenTimeKey(source, group, artifact, version) - if !store { - return ctx, false - } - if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return ctx, false - } - return WithUpstreamLastModified(ctx, func(published time.Time) { - if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, published); err != nil { - h.proxy.Logger.Warn("cooldown: could not store maven publish time", - "group", group, "artifact", artifact, "version", version, "error", err) - } - }), true -} - -// pomLastModified sends a HEAD request for a .pom and returns its -// Last-Modified. found is false when upstream doesn't have the file. Only a -// 2xx response's header is used: an error page's date says nothing about the -// version. -func (h *MavenHandler) pomLastModified(ctx context.Context, pomURL string) (published time.Time, found bool, err error) { - req, err := http.NewRequestWithContext(ctx, http.MethodHead, pomURL, nil) - if err != nil { - return time.Time{}, false, err - } - // Name the proxy as its artifact downloads do; Maven Central asks - // tools not to send an HTTP library's default. - req.Header.Set("User-Agent", mavenTimeUserAgent) - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return time.Time{}, false, fmt.Errorf("reading pom time: %w", err) - } - _ = resp.Body.Close() - switch { - case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: - return time.Time{}, false, nil - case resp.StatusCode == http.StatusTooManyRequests && h.proxy.RateLimits != nil: - // The gate has just noted the upstream's Retry-After; pass it on. - if err := h.proxy.RateLimits.Held(pomURL); err != nil { - return time.Time{}, false, err - } - return time.Time{}, false, fmt.Errorf("reading pom time: upstream returned %d", resp.StatusCode) - case resp.StatusCode < 200 || resp.StatusCode > 299: - return time.Time{}, false, fmt.Errorf("reading pom time: upstream returned %d", resp.StatusCode) - } - published, err = http.ParseTime(resp.Header.Get(headerLastModified)) - if err != nil { - return time.Time{}, true, nil - } - return published, true, nil -} - -// checkCooldown answers a request for a file of a version in cooldown, or -// whose time couldn't be read, and reports whether it did. The answer is -// 403, as for a version on the denylist. -func (h *MavenHandler) checkCooldown(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath, version string) bool { - if !h.cooldownEnabled() || strings.HasSuffix(version, mavenSnapshot) { - return false - } - artifactDir := path.Dir(path.Dir(urlPath)) - group, artifact, ok := mavenCoordinates(artifactDir) - if !ok || h.proxy.Cooldown.For("maven", mavenPackagePURL(group, artifact)) == 0 { - return false - } - published, err := h.publishTime(r.Context(), source, artifactDir, version) - if err != nil { - if writeUpstreamHeld(w, err) { - return true - } - h.proxy.Logger.Warn("cooldown: could not read maven version time", - "group", group, "artifact", artifact, "version", version, "error", err) - http.Error(w, "failed to check cooldown", http.StatusBadGateway) - return true - } - if !h.proxy.cooldown().IsAllowed("maven", mavenPackagePURL(group, artifact), published) { - h.proxy.Logger.Info("cooldown: withholding maven version", - "group", group, "artifact", artifact, "version", version) - http.Error(w, errMavenCooldown+": "+canonicalVersionPURL("maven", group+":"+artifact, version), http.StatusForbidden) - return true - } - return false -} - -// serveCooldownMetadata serves maven-metadata.xml without versions in -// cooldown, and its checksum files computed from what is served. It reports -// false for other files beside the metadata, such as a signature, when the -// metadata is served unchanged, so they are proxied as usual. -func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) bool { - filename := path.Base(urlPath) - suffix, ok := strings.CutPrefix(filename, mavenMetadataFile) - if !ok { - return false - } - metadataPath := path.Join(path.Dir(urlPath), mavenMetadataFile) - body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, metadataPath) - if err != nil { - h.writeMetadataError(w, err) - return true - } - - m, err := h.proxy.cachedRewrite(r.Context(), "maven", h.proxyURL, cacheKey, body, "", func(in []byte) ([]byte, []string, error) { - // The rewrite is shared by requests waiting on it and cached, so it - // finishes even if the request that started it goes away. - out, err := h.filterMetadata(context.WithoutCancel(r.Context()), source, path.Dir(urlPath), in) - return out, nil, err - }) - if err != nil { - if r.Context().Err() != nil { - return true - } - if writeUpstreamHeld(w, err) { - return true - } - h.proxy.Logger.Warn("cooldown: could not filter maven metadata", "path", metadataPath, "error", err) - http.Error(w, "failed to check cooldown", http.StatusBadGateway) - return true - } - - switch { - case suffix == "": - // The metadata cache records JSON when an upstream sent no type, - // and maven-metadata.xml is always XML. - if contentType == "" || contentType == contentTypeJSON { - contentType = "text/xml" - } - markStale(w, h.proxy.lookupCachedMeta("maven", cacheKey).stale) - serveRewrittenMetadataAs(w, r, m, contentType) - case mavenChecksums[suffix] != nil: - sum := mavenChecksums[suffix]() - _, _ = sum.Write(m.content()) - w.Header().Set(headerContentType, "text/plain") - _, _ = io.WriteString(w, hex.EncodeToString(sum.Sum(nil))) - case bytes.Equal(m.content(), body): - return false - default: - // A signature of the upstream metadata doesn't match what is served. - http.Error(w, "not found", http.StatusNotFound) - } - return true -} - -// filterMetadata removes versions in cooldown from an artifact's -// maven-metadata.xml and points and at the newest -// versions left. Metadata without a version list, such as a group's plugin -// list or a SNAPSHOT's build list, is returned unchanged. The document is -// edited in place rather than encoded again, so everything else in it stays -// as upstream sent it. -func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, artifactDir string, body []byte) ([]byte, error) { - doc, err := parseMavenMetadata(body) - if err != nil { - return nil, err - } - group, artifact, ok := mavenCoordinates(artifactDir) - if !ok || len(doc.versions) == 0 { - return body, nil - } - packagePURL := mavenPackagePURL(group, artifact) - // No version can be newer than the metadata listing it. - if lastUpdated, err := time.Parse(mavenLastUpdatedLayout, doc.lastUpdated); err == nil && - h.proxy.cooldown().IsAllowed("maven", packagePURL, lastUpdated) { - return body, nil - } - - removed := map[string]bool{} - for i := len(doc.versions) - 1; i >= 0; i-- { - v := doc.versions[i].value - if strings.HasSuffix(v, mavenSnapshot) { - continue - } - published, err := h.publishTime(ctx, source, artifactDir, v) - if err != nil { - return nil, fmt.Errorf("reading time of %s: %w", v, err) - } - if published.IsZero() { - continue - } - if h.proxy.cooldown().IsAllowed("maven", packagePURL, published) { - break - } - h.proxy.Logger.Info("cooldown: filtering maven version", "group", group, "artifact", artifact, "version", v) - removed[v] = true - } - if len(removed) == 0 { - return body, nil - } - return doc.without(body, removed), nil -} - -// mavenElement is an element of maven-metadata.xml: its text and where it -// starts and ends in the document. -type mavenElement struct { - value string - start, end int -} - -type mavenMetadata struct { - versions []mavenElement - latest, release *mavenElement - lastUpdated string -} - -// parseMavenMetadata reads the version list, , and -// of a maven-metadata.xml, with their offsets. -func parseMavenMetadata(body []byte) (*mavenMetadata, error) { - dec := xml.NewDecoder(bytes.NewReader(body)) - // Versions are ASCII, so any declared charset can be read as bytes. - dec.CharsetReader = func(_ string, in io.Reader) (io.Reader, error) { return in, nil } - - doc := &mavenMetadata{} - var stack []string - var current *mavenElement - for { - offset := int(dec.InputOffset()) - tok, err := dec.Token() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return nil, fmt.Errorf("parsing maven metadata: %w", err) - } - switch t := tok.(type) { - case xml.StartElement: - stack = append(stack, t.Name.Local) - switch strings.Join(stack, "/") { - case "metadata/versioning/versions/version", - "metadata/versioning/latest", - "metadata/versioning/release", - "metadata/versioning/lastUpdated": - current = &mavenElement{start: offset} - } - case xml.CharData: - if current != nil { - current.value += string(t) - } - case xml.EndElement: - if current != nil { - current.value = strings.TrimSpace(current.value) - current.end = int(dec.InputOffset()) - switch t.Name.Local { - case "version": - doc.versions = append(doc.versions, *current) - case "latest": - doc.latest = current - case "release": - doc.release = current - case "lastUpdated": - doc.lastUpdated = current.value - } - current = nil - } - if len(stack) > 0 { - stack = stack[:len(stack)-1] - } - } - } - return doc, nil -} - -// without returns body with the removed versions taken out of the version -// list, and and set to the last versions left in list -// order, which is how the repository chose them. Either is dropped when no -// version is left for it. -func (doc *mavenMetadata) without(body []byte, removed map[string]bool) []byte { - type edit struct { - start, end int - text string - } - var edits []edit - drop := func(e mavenElement) { - edits = append(edits, edit{start: lineStart(body, e.start), end: e.end}) - } - var latest, release string - for _, v := range doc.versions { - if removed[v.value] { - drop(v) - continue - } - latest = v.value - if !strings.HasSuffix(v.value, mavenSnapshot) { - release = v.value - } - } - for _, field := range []struct { - el *mavenElement - name string - next string - }{{doc.latest, "latest", latest}, {doc.release, "release", release}} { - if field.el == nil || !removed[field.el.value] { - continue - } - if field.next == "" { - drop(*field.el) - continue - } - edits = append(edits, edit{field.el.start, field.el.end, "<" + field.name + ">" + field.next + ""}) - } - - // Edits don't overlap; apply them in document order. - slices.SortFunc(edits, func(a, b edit) int { return a.start - b.start }) - out := make([]byte, 0, len(body)) - pos := 0 - for _, e := range edits { - out = append(out, body[pos:e.start]...) - out = append(out, e.text...) - pos = e.end - } - return append(out, body[pos:]...) -} - -// lineStart moves an element's start back over the indentation and line -// break before it, so removing the element removes its line. -func lineStart(body []byte, start int) int { - i := start - for i > 0 && (body[i-1] == ' ' || body[i-1] == '\t') { - i-- - } - if i > 0 && body[i-1] == '\n' { - i-- - if i > 0 && body[i-1] == '\r' { - i-- - } - return i - } - return start -} diff --git a/internal/handler/maven_cooldown_test.go b/internal/handler/maven_cooldown_test.go deleted file mode 100644 index 828d1ff..0000000 --- a/internal/handler/maven_cooldown_test.go +++ /dev/null @@ -1,530 +0,0 @@ -package handler - -import ( - "compress/gzip" - "crypto/md5" - "crypto/sha1" - "crypto/sha256" - "crypto/sha512" - "encoding/hex" - "fmt" - "hash" - "io" - "net/http" - "net/http/httptest" - "slices" - "strings" - "sync" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - registryclient "github.com/git-pkgs/registries/client" - "github.com/git-pkgs/registries/fetch" -) - -const mavenTestDir = "/com/example/lib/" - -// mavenCooldownUpstream serves com.example:lib. Each version's .pom has the -// Last-Modified in times; pomStatus overrides the status of a version's -// .pom. Requests for .pom files are counted. -type mavenCooldownUpstream struct { - server *httptest.Server - versions []string - latest string - release string - lastUpdated time.Time - times map[string]time.Time - pomStatus map[string]int - - mu sync.Mutex - poms map[string]int - heads int - userAgents []string -} - -func newMavenCooldownUpstream(t *testing.T) *mavenCooldownUpstream { - t.Helper() - now := time.Now() - u := &mavenCooldownUpstream{ - versions: []string{"1.0", "1.1", "2.0", "2.1", "3.0-SNAPSHOT"}, - latest: "3.0-SNAPSHOT", - release: "2.1", - lastUpdated: now.Add(-time.Hour), - times: map[string]time.Time{ - "1.0": now.Add(-30 * 24 * time.Hour), - "1.1": now.Add(-10 * 24 * time.Hour), - "2.0": now.Add(-2 * time.Hour), - "2.1": now.Add(-time.Hour), - "3.0-SNAPSHOT": now.Add(-time.Minute), - }, - pomStatus: map[string]int{}, - poms: map[string]int{}, - } - u.server = httptest.NewServer(http.HandlerFunc(u.serve)) - t.Cleanup(u.server.Close) - return u -} - -// serve answers for the default upstream and, under /portal, for the -// Gradle Plugin Portal fallback alike. -func (u *mavenCooldownUpstream) serve(w http.ResponseWriter, r *http.Request) { - rest, ok := strings.CutPrefix(strings.TrimPrefix(r.URL.Path, "/portal"), mavenTestDir) - if !ok { - http.NotFound(w, r) - return - } - switch rest { - case "maven-metadata.xml": - _, _ = w.Write([]byte(u.metadata())) - return - case "maven-metadata.xml.asc": - _, _ = w.Write([]byte("signature")) - return - } - version, file, _ := strings.Cut(rest, "/") - published, known := u.times[version] - if !known { - http.NotFound(w, r) - return - } - if strings.HasSuffix(file, ".module") { - w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) - } - if strings.HasSuffix(file, ".pom") { - u.mu.Lock() - u.poms[version]++ - if r.Method == http.MethodHead { - u.heads++ - u.userAgents = append(u.userAgents, r.UserAgent()) - } - u.mu.Unlock() - w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) - if status := u.pomStatus[version]; status != 0 { - // An error page dated long ago must not count as an old version. - w.Header().Set(headerLastModified, time.Unix(0, 0).UTC().Format(http.TimeFormat)) - if status == http.StatusTooManyRequests { - w.Header().Set("Retry-After", "60") - } - w.WriteHeader(status) - return - } - } - _, _ = w.Write([]byte(file + " of " + version)) -} - -func (u *mavenCooldownUpstream) metadata() string { - var b strings.Builder - b.WriteString("\n\n com.example\n lib\n \n") - b.WriteString(" " + u.latest + "\n " + u.release + "\n \n") - for _, v := range u.versions { - b.WriteString(" " + v + "\n") - } - b.WriteString(" \n " + u.lastUpdated.UTC().Format(mavenLastUpdatedLayout) + "\n \n\n") - return b.String() -} - -// pomRequests returns how many requests each version's .pom got. -func (u *mavenCooldownUpstream) pomRequests() map[string]int { - u.mu.Lock() - defer u.mu.Unlock() - out := make(map[string]int, len(u.poms)) - for v, n := range u.poms { - out[v] = n - } - return out -} - -func (u *mavenCooldownUpstream) totalPOMRequests() int { - total := 0 - for _, n := range u.pomRequests() { - total += n - } - return total -} - -func newMavenCooldownHandler(t *testing.T, u *mavenCooldownUpstream) (http.Handler, *Proxy) { - t.Helper() - proxy, _, store, _ := setupTestProxy(t) - store.seekable = true - proxy.HTTPClient = u.server.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(u.server.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - return NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal").Routes(), proxy -} - -func mavenMetadataVersions(t *testing.T, body string) []string { - t.Helper() - doc, err := parseMavenMetadata([]byte(body)) - if err != nil { - t.Fatalf("served metadata does not parse: %v\n%s", err, body) - } - var versions []string - for _, v := range doc.versions { - versions = append(versions, v.value) - } - return versions -} - -func TestMavenCooldownFiltersMetadata(t *testing.T) { - u := newMavenCooldownUpstream(t) - h, proxy := newMavenCooldownHandler(t, u) - - w := getMaven(h, mavenTestDir+"maven-metadata.xml") - if w.Code != http.StatusOK { - t.Fatalf("status = %d: %s", w.Code, w.Body) - } - body := w.Body.String() - if got := strings.Join(mavenMetadataVersions(t, body), " "); got != "1.0 1.1 3.0-SNAPSHOT" { - t.Errorf("versions = %s, want 1.0 1.1 3.0-SNAPSHOT", got) - } - if !strings.Contains(body, "1.1") || !strings.Contains(body, "3.0-SNAPSHOT") { - t.Errorf("release or latest not recomputed:\n%s", body) - } - if strings.Contains(body, "\n\n") { - t.Errorf("removed versions left blank lines:\n%s", body) - } - - // The walk stops at 1.1, the first version old enough, and skips the - // SNAPSHOT. - want := map[string]int{"2.1": 1, "2.0": 1, "1.1": 1} - if got := u.pomRequests(); len(got) != len(want) || got["2.1"] != 1 || got["2.0"] != 1 || got["1.1"] != 1 { - t.Errorf("pom requests = %v, want %v", got, want) - } - - // Times are stored, so the next request makes no .pom requests. - ver, err := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")) - if err != nil || ver == nil || !ver.PublishedAt.Valid { - t.Fatalf("time of 2.1 not stored: %v %v", ver, err) - } - if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != body { - t.Errorf("second response differs:\n%s", w.Body) - } - if n := u.totalPOMRequests(); n != 3 { - t.Errorf("pom requests after second metadata request = %d, want 3", n) - } -} - -func TestMavenCooldownRecomputesLatest(t *testing.T) { - u := newMavenCooldownUpstream(t) - u.versions = []string{"1.0", "1.1", "2.0", "2.1"} - u.latest = "2.1" - h, _ := newMavenCooldownHandler(t, u) - - body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String() - if !strings.Contains(body, "1.1") || !strings.Contains(body, "1.1") { - t.Errorf("latest and release should be 1.1:\n%s", body) - } -} - -func TestMavenCooldownDropsLatestWhenNothingIsLeft(t *testing.T) { - u := newMavenCooldownUpstream(t) - u.versions = []string{"2.0", "2.1"} - u.latest = "2.1" - h, _ := newMavenCooldownHandler(t, u) - - body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String() - if strings.Contains(body, "") || strings.Contains(body, "") || strings.Contains(body, "") { - t.Errorf("no version is out of cooldown:\n%s", body) - } - if !strings.Contains(body, "\n ") { - t.Errorf("version list not emptied in place:\n%s", body) - } -} - -func TestMavenCooldownChecksumsMatchMetadata(t *testing.T) { - u := newMavenCooldownUpstream(t) - h, _ := newMavenCooldownHandler(t, u) - - body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.Bytes() - for suffix, newHash := range map[string]func() hash.Hash{ - ".md5": md5.New, ".sha1": sha1.New, ".sha256": sha256.New, ".sha512": sha512.New, - } { - sum := newHash() - _, _ = sum.Write(body) - w := getMaven(h, mavenTestDir+"maven-metadata.xml"+suffix) - if w.Code != http.StatusOK || w.Body.String() != hex.EncodeToString(sum.Sum(nil)) { - t.Errorf("%s: status = %d body %q, want checksum of served metadata", suffix, w.Code, w.Body) - } - } - // A signature of the upstream metadata would not match. - if w := getMaven(h, mavenTestDir+"maven-metadata.xml.asc"); w.Code != http.StatusNotFound { - t.Errorf(".asc status = %d, want 404", w.Code) - } - - // Unchanged metadata keeps its signature. - u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) - if w := getMaven(h, mavenTestDir+"maven-metadata.xml.asc"); w.Code != http.StatusOK || w.Body.String() != "signature" { - t.Errorf("unchanged .asc: status = %d body %q, want upstream's", w.Code, w.Body) - } -} - -func TestMavenCooldownRefusesFiles(t *testing.T) { - u := newMavenCooldownUpstream(t) - h, _ := newMavenCooldownHandler(t, u) - - for _, file := range []string{"2.1/lib-2.1.pom", "2.1/lib-2.1.jar", "2.1/lib-2.1.jar.sha1", "2.0/lib-2.0-sources.jar"} { - w := getMaven(h, mavenTestDir+file) - if w.Code != http.StatusForbidden || !strings.Contains(w.Body.String(), errMavenCooldown) { - t.Errorf("%s: status = %d body %q, want 403 cooldown", file, w.Code, w.Body) - } - } - for _, file := range []string{"1.1/lib-1.1.jar", "3.0-SNAPSHOT/lib-3.0-SNAPSHOT.jar"} { - if w := getMaven(h, mavenTestDir+file); w.Code != http.StatusOK { - t.Errorf("%s: status = %d, want 200", file, w.Code) - } - } - // One .pom request per version, none for the SNAPSHOT. - if got := u.pomRequests(); got["2.1"] != 1 || got["2.0"] != 1 || got["1.1"] != 1 || got["3.0-SNAPSHOT"] != 0 { - t.Errorf("pom requests = %v", got) - } -} - -// TestMavenCooldownPOMDownloadCarriesItsTime checks that a build fetching -// pinned versions, which never reads maven-metadata.xml, costs no requests -// beyond its downloads: the .pom's own Last-Modified gives the version's -// time, and Maven fetches the .pom before the version's other files. -func TestMavenCooldownPOMDownloadCarriesItsTime(t *testing.T) { - u := newMavenCooldownUpstream(t) - h, proxy := newMavenCooldownHandler(t, u) - - for _, c := range []struct { - file string - want int - }{ - {"2.1/lib-2.1.pom", http.StatusForbidden}, - {"2.1/lib-2.1.jar", http.StatusForbidden}, - {"1.0/lib-1.0.pom", http.StatusOK}, - {"1.0/lib-1.0.jar", http.StatusOK}, - } { - if w := getMaven(h, mavenTestDir+c.file); w.Code != c.want { - t.Errorf("%s: status = %d, want %d", c.file, w.Code, c.want) - } - } - u.mu.Lock() - heads := u.heads - u.mu.Unlock() - if heads != 0 { - t.Errorf("HEAD requests = %d, want 0", heads) - } - if got := u.pomRequests(); got["2.1"] != 1 || got["1.0"] != 1 { - t.Errorf("pom requests = %v, want one GET each", got) - } - ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")) - if ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(u.times["2.1"].Truncate(time.Second)) { - t.Errorf("stored time of 2.1 = %v, want %v", ver, u.times["2.1"]) - } -} - -// TestMavenCooldownModuleDownloadCarriesItsTime checks that a Gradle -// .module fetched before the .pom gives the version's time as the .pom -// would. -func TestMavenCooldownModuleDownloadCarriesItsTime(t *testing.T) { - u := newMavenCooldownUpstream(t) - h, _ := newMavenCooldownHandler(t, u) - - if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.module"); w.Code != http.StatusForbidden { - t.Errorf("2.1 .module: status = %d, want 403", w.Code) - } - if w := getMaven(h, mavenTestDir+"1.0/lib-1.0.module"); w.Code != http.StatusOK { - t.Errorf("1.0 .module: status = %d, want 200", w.Code) - } - if w := getMaven(h, mavenTestDir+"1.0/lib-1.0.jar"); w.Code != http.StatusOK { - t.Errorf("1.0 .jar: status = %d, want 200", w.Code) - } - if n := u.totalPOMRequests(); n != 0 { - t.Errorf("pom requests = %d, want 0", n) - } -} - -// TestMavenCooldownTimeRequestsNameTheProxy checks the User-Agent of the -// HEAD requests for publish times: Maven Central asks tools to name -// themselves rather than send an HTTP library's default. -func TestMavenCooldownTimeRequestsNameTheProxy(t *testing.T) { - u := newMavenCooldownUpstream(t) - h, _ := newMavenCooldownHandler(t, u) - - getMaven(h, mavenTestDir+"maven-metadata.xml") - u.mu.Lock() - defer u.mu.Unlock() - if len(u.userAgents) == 0 { - t.Fatal("no HEAD requests") - } - for _, ua := range u.userAgents { - if ua != mavenTimeUserAgent { - t.Errorf("User-Agent = %q, want %q", ua, mavenTimeUserAgent) - } - } -} - -// TestMavenCooldownMetadataServedLikeOtherRewrites checks that filtered -// metadata is gzip-compressed for clients that accept it, as the other -// ecosystems' rewritten metadata is. -func TestMavenCooldownMetadataServedLikeOtherRewrites(t *testing.T) { - u := newMavenCooldownUpstream(t) - // Enough versions that the document compresses. - for i := range 50 { - v := fmt.Sprintf("0.%d", i) - u.versions = append([]string{v}, u.versions...) - u.times[v] = time.Now().Add(-100 * 24 * time.Hour) - } - h, proxy := newMavenCooldownHandler(t, u) - // The server always sets up the rewrite cache, which keeps the gzip copy. - proxy.SetMetadataRewriteCacheSize(1 << 20) - - req := httptest.NewRequest(http.MethodGet, mavenTestDir+"maven-metadata.xml", nil) - req.Header.Set(headerAcceptEncoding, "gzip") - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != http.StatusOK || w.Header().Get(headerContentEncoding) != "gzip" { - t.Fatalf("status = %d, Content-Encoding %q; want gzip", w.Code, w.Header().Get(headerContentEncoding)) - } - if ct := w.Header().Get(headerContentType); !strings.Contains(ct, "xml") { - t.Errorf("Content-Type = %q, want XML", ct) - } - zr, err := gzip.NewReader(w.Body) - if err != nil { - t.Fatal(err) - } - body, _ := io.ReadAll(zr) - if versions := mavenMetadataVersions(t, string(body)); slices.Contains(versions, "2.1") { - t.Errorf("2.1 listed in the gzip body: %v", versions) - } -} - -func TestMavenCooldownOldLastUpdatedSkipsPOMs(t *testing.T) { - u := newMavenCooldownUpstream(t) - u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) - // The times would filter 2.0 and 2.1 if they were read. - h, _ := newMavenCooldownHandler(t, u) - - w := getMaven(h, mavenTestDir+"maven-metadata.xml") - if w.Body.String() != u.metadata() { - t.Errorf("metadata changed:\n%s", w.Body) - } - if n := u.totalPOMRequests(); n != 0 { - t.Errorf("pom requests = %d, want 0", n) - } -} - -func TestMavenCooldownIgnoresNon2xxPOMResponses(t *testing.T) { - u := newMavenCooldownUpstream(t) - u.pomStatus["2.1"] = http.StatusNotFound - h, proxy := newMavenCooldownHandler(t, u) - - // A missing .pom gives no time: 2.1 stays listed, and the walk goes on - // to 2.0 rather than stopping at the error page's old date. - versions := mavenMetadataVersions(t, getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String()) - if got := strings.Join(versions, " "); got != "1.0 1.1 2.1 3.0-SNAPSHOT" { - t.Errorf("versions = %s, want 1.0 1.1 2.1 3.0-SNAPSHOT", got) - } - if ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")); ver != nil && ver.PublishedAt.Valid { - t.Errorf("time of a 404 stored: %v", ver.PublishedAt.Time) - } - - // Any other error fails closed. - u.pomStatus["2.1"] = http.StatusTooManyRequests - for _, p := range []string{"maven-metadata.xml", "maven-metadata.xml.sha1", "2.1/lib-2.1.jar"} { - if w := getMaven(h, mavenTestDir+p); w.Code != http.StatusBadGateway { - t.Errorf("%s: status = %d, want 502", p, w.Code) - } - } - if ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")); ver != nil && ver.PublishedAt.Valid { - t.Errorf("time of a 429 stored: %v", ver.PublishedAt.Time) - } -} - -// TestMavenCooldownRateLimitedPOMIsHeld checks that a 429 on a .pom time -// request is answered like any held upstream request, with 503 and the -// upstream's Retry-After, so clients wait instead of failing the build. -func TestMavenCooldownRateLimitedPOMIsHeld(t *testing.T) { - u := newMavenCooldownUpstream(t) - u.pomStatus["2.0"] = http.StatusTooManyRequests - h, proxy := newMavenCooldownHandler(t, u) - gate := registryclient.NewRateLimitGate(u.server.Client().Transport) - proxy.HTTPClient = &http.Client{Transport: gate} - proxy.RateLimits = gate - - // Not the .pom, so its time is read with a HEAD. - w := getMaven(h, mavenTestDir+"2.0/lib-2.0.jar") - if w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" { - t.Errorf("status = %d, Retry-After %q; want 503 with Retry-After", w.Code, w.Header().Get("Retry-After")) - } -} - -func TestMavenCooldownDisabledPassesThrough(t *testing.T) { - u := newMavenCooldownUpstream(t) - h, proxy := newMavenCooldownHandler(t, u) - proxy.Cooldown = nil - - if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != u.metadata() { - t.Errorf("metadata changed:\n%s", w.Body) - } - if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusOK { - t.Errorf("jar status = %d, want 200", w.Code) - } - if n := u.totalPOMRequests(); n != 0 { - t.Errorf("pom requests = %d, want 0", n) - } -} - -func TestMavenCooldownPackageOverride(t *testing.T) { - u := newMavenCooldownUpstream(t) - h, proxy := newMavenCooldownHandler(t, u) - proxy.Cooldown = &cooldown.Config{Default: "3d", Packages: map[string]string{"pkg:maven/com.example/lib": "0"}} - - if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != u.metadata() { - t.Errorf("metadata changed:\n%s", w.Body) - } - if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusOK { - t.Errorf("jar status = %d, want 200", w.Code) - } - if n := u.totalPOMRequests(); n != 0 { - t.Errorf("pom requests = %d, want 0", n) - } -} - -func TestMavenCooldownNamedRepositoryStoresItsOwnTimes(t *testing.T) { - u := newMavenCooldownUpstream(t) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = u.server.Client() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - // The named repository is the same server under /portal, which serves - // the same files. - h := NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal"). - WithRepositories(map[string]string{"mirror": u.server.URL + "/portal"}).Routes() - - for range 2 { - versions := mavenMetadataVersions(t, getMaven(h, "/mirror"+mavenTestDir+"maven-metadata.xml").Body.String()) - if got := strings.Join(versions, " "); got != "1.0 1.1 3.0-SNAPSHOT" { - t.Fatalf("versions = %s, want 1.0 1.1 3.0-SNAPSHOT", got) - } - } - if w := getMaven(h, "/mirror"+mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusForbidden { - t.Errorf("jar status = %d, want 403", w.Code) - } - if n := u.totalPOMRequests(); n != 3 { - t.Errorf("pom requests = %d, want 3", n) - } - - // The default repository doesn't use the named one's times. - _ = getMaven(h, mavenTestDir+"maven-metadata.xml") - if n := u.totalPOMRequests(); n != 6 { - t.Errorf("pom requests after default metadata = %d, want 6", n) - } -} - -func TestMavenMetadataWithoutCompact(t *testing.T) { - in := `2212` - doc, err := parseMavenMetadata([]byte(in)) - if err != nil { - t.Fatal(err) - } - got := string(doc.without([]byte(in), map[string]bool{"2": true})) - want := `111` - if got != want { - t.Errorf("got %s\nwant %s", got, want) - } -} diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go deleted file mode 100644 index e9b2fcf..0000000 --- a/internal/handler/maven_repositories_test.go +++ /dev/null @@ -1,225 +0,0 @@ -package handler - -import ( - "io" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/registries/fetch" -) - -// mavenRepoUpstream serves label-specific bytes for every path it knows and -// counts its requests. -func mavenRepoUpstream(t *testing.T, label string, known ...string) (*httptest.Server, *atomic.Int32) { - t.Helper() - var requests atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - for _, p := range known { - if r.URL.Path == p { - _, _ = io.WriteString(w, label+" "+r.URL.Path) - return - } - } - http.NotFound(w, r) - })) - t.Cleanup(srv.Close) - return srv, &requests -} - -func newMavenRepositoriesHandler(t *testing.T, central, portal, grails *httptest.Server) http.Handler { - t.Helper() - proxy, _, store, _ := setupTestProxy(t) - store.seekable = true - proxy.HTTPClient = central.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(central.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - return NewMavenHandler(proxy, "http://localhost", central.URL, portal.URL). - WithRepositories(map[string]string{"grails": grails.URL + "/restricted/"}).Routes() -} - -func getMaven(h http.Handler, path string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - return w -} - -const ( - mavenJarPath = "/org/grails/grails-core/6.0.0/grails-core-6.0.0.jar" - mavenMetadataPath = "/org/grails/grails-core/maven-metadata.xml" -) - -func TestMavenRepositoriesAreServedFromTheirOwnUpstream(t *testing.T) { - central, _ := mavenRepoUpstream(t, "central", mavenJarPath, mavenMetadataPath) - portal, _ := mavenRepoUpstream(t, "portal") - grails, _ := mavenRepoUpstream(t, "grails", "/restricted"+mavenJarPath, "/restricted"+mavenMetadataPath) - h := newMavenRepositoriesHandler(t, central, portal, grails) - - // The same coordinates in both repositories are cached apart. - for range 2 { - if w := getMaven(h, "/grails"+mavenJarPath); w.Code != http.StatusOK || w.Body.String() != "grails /restricted"+mavenJarPath { - t.Fatalf("named jar: status = %d, body = %q", w.Code, w.Body) - } - if w := getMaven(h, mavenJarPath); w.Code != http.StatusOK || w.Body.String() != "central "+mavenJarPath { - t.Fatalf("default jar: status = %d, body = %q", w.Code, w.Body) - } - } - - if w := getMaven(h, "/grails"+mavenMetadataPath); w.Code != http.StatusOK || !strings.HasPrefix(w.Body.String(), "grails ") { - t.Errorf("named metadata: status = %d, body = %q", w.Code, w.Body) - } - if w := getMaven(h, mavenMetadataPath); w.Code != http.StatusOK || !strings.HasPrefix(w.Body.String(), "central ") { - t.Errorf("default metadata: status = %d, body = %q", w.Code, w.Body) - } -} - -func TestMavenRepositoriesDoNotFallBackToThePluginPortal(t *testing.T) { - const pluginPath = "/org/example/plugin/1.0/plugin-1.0.jar" - central, _ := mavenRepoUpstream(t, "central") - portal, portalRequests := mavenRepoUpstream(t, "portal", pluginPath, "/org/example/plugin/maven-metadata.xml") - grails, _ := mavenRepoUpstream(t, "grails") - h := newMavenRepositoriesHandler(t, central, portal, grails) - - for _, path := range []string{"/grails" + pluginPath, "/grails/org/example/plugin/maven-metadata.xml"} { - if w := getMaven(h, path); w.Code != http.StatusNotFound { - t.Errorf("GET %s status = %d, want 404", path, w.Code) - } - } - if n := portalRequests.Load(); n != 0 { - t.Errorf("named repository asked the plugin portal %d times", n) - } - if w := getMaven(h, pluginPath); w.Code != http.StatusOK || !strings.HasPrefix(w.Body.String(), "portal ") { - t.Errorf("default repository: status = %d, body = %q, want the plugin portal's copy", w.Code, w.Body) - } -} - -func TestMavenRepositoryRootIsNotFound(t *testing.T) { - central, centralRequests := mavenRepoUpstream(t, "central") - portal, _ := mavenRepoUpstream(t, "portal") - grails, _ := mavenRepoUpstream(t, "grails") - h := newMavenRepositoriesHandler(t, central, portal, grails) - - for _, path := range []string{"/grails", "/grails/"} { - if w := getMaven(h, path); w.Code != http.StatusNotFound { - t.Errorf("GET %s status = %d, want 404", path, w.Code) - } - } - if n := centralRequests.Load(); n != 0 { - t.Errorf("repository root went to the default upstream %d times", n) - } -} - -func TestMavenRepositoryRepointedServesTheNewUpstream(t *testing.T) { - central, _ := mavenRepoUpstream(t, "central") - portal, _ := mavenRepoUpstream(t, "portal") - staging, _ := mavenRepoUpstream(t, "staging", mavenJarPath) - production, _ := mavenRepoUpstream(t, "production", mavenJarPath) - - proxy, _, store, _ := setupTestProxy(t) - store.seekable = true - proxy.HTTPClient = central.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(central.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - - for _, upstream := range []struct { - srv *httptest.Server - label string - }{{staging, "staging"}, {production, "production"}} { - h := NewMavenHandler(proxy, "http://localhost", central.URL, portal.URL). - WithRepositories(map[string]string{"grails": upstream.srv.URL}).Routes() - if w := getMaven(h, "/grails"+mavenJarPath); w.Code != http.StatusOK || w.Body.String() != upstream.label+" "+mavenJarPath { - t.Errorf("%s: status = %d, body = %q", upstream.label, w.Code, w.Body) - } - } -} - -// TestMavenDottedGroupDirectoryNotCachedAsGroup checks that a file whose -// group directories contain dots or colons, which the Maven layout never -// has, is not served: it would either share the cache entry of the group -// the dots spell or be proxied past the denylist, scan and cooldown. -func TestMavenDottedGroupDirectoryNotCachedAsGroup(t *testing.T) { - const standard = "/org/foo/bar/1.0/bar-1.0.jar" - odd := []string{ - "/org.foo/bar/1.0/bar-1.0.jar", "/org.foo/bar/1.0/bar-1.0-linux.exe", - "/org/foo:x/bar/1.0/bar-1.0.jar", "/org/foo/b:ar/1.0/b:ar-1.0.jar", - } - central, requests := mavenRepoUpstream(t, "central", append([]string{standard}, odd...)...) - portal, _ := mavenRepoUpstream(t, "portal") - var restricted []string - for _, p := range odd { - restricted = append(restricted, "/restricted"+p) - } - grails, grailsRequests := mavenRepoUpstream(t, "grails", restricted...) - h := newMavenRepositoriesHandler(t, central, portal, grails) - - if w := getMaven(h, standard); w.Code != http.StatusOK || w.Body.String() != "central "+standard { - t.Errorf("GET %s = %d %q", standard, w.Code, w.Body.String()) - } - before := requests.Load() + grailsRequests.Load() - for _, p := range odd { - for _, path := range []string{p, "/grails" + p} { - if w := getMaven(h, path); w.Code != http.StatusNotFound { - t.Errorf("GET %s = %d %q, want 404", path, w.Code, w.Body.String()) - } - } - } - if n := requests.Load() + grailsRequests.Load() - before; n != 0 { - t.Errorf("%d upstream requests for paths outside the Maven layout", n) - } -} - -// TestMavenUncachedPathFallsBackToPluginPortal checks that a path proxied -// without caching, such as a catalog, still tries the Gradle Plugin Portal -// when the primary upstream lacks it. -func TestMavenUncachedPathFallsBackToPluginPortal(t *testing.T) { - const uncached = "/org/foo/archetype-catalog.xml" - central, _ := mavenRepoUpstream(t, "central") - portal, _ := mavenRepoUpstream(t, "portal", uncached) - grails, _ := mavenRepoUpstream(t, "grails") - h := newMavenRepositoriesHandler(t, central, portal, grails) - - if w := getMaven(h, uncached); w.Code != http.StatusOK || w.Body.String() != "portal "+uncached { - t.Errorf("GET %s = %d %q, want the Plugin Portal's copy", uncached, w.Code, w.Body.String()) - } -} - -func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { - const uncached = "/org/foo/archetype-catalog.xml" - ranged := func() *httptest.Server { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("ETag", `"v2"`) - http.ServeContent(w, r, "archetype-catalog.xml", time.Time{}, strings.NewReader("0123456789")) - })) - t.Cleanup(srv.Close) - return srv - } - central, grails := ranged(), ranged() - portal, _ := mavenRepoUpstream(t, "portal") - h := newMavenRepositoriesHandler(t, central, portal, grails) - - for _, p := range []string{uncached, "/grails" + uncached} { - req := httptest.NewRequest(http.MethodGet, p, nil) - req.Header.Set("Range", "bytes=2-4") - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != http.StatusPartialContent || w.Body.String() != "234" { - t.Errorf("GET %s with Range = %d %q, want 206 %q", p, w.Code, w.Body.String(), "234") - } - - // A Range for an older copy gets the whole current file. - req = httptest.NewRequest(http.MethodGet, p, nil) - req.Header.Set("Range", "bytes=2-4") - req.Header.Set("If-Range", `"v1"`) - w = httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != http.StatusOK || w.Body.String() != "0123456789" { - t.Errorf("GET %s with stale If-Range = %d %q, want 200 with the whole file", p, w.Code, w.Body.String()) - } - } -} diff --git a/internal/handler/maven_test.go b/internal/handler/maven_test.go index 14cb49f..9ca5eb6 100644 --- a/internal/handler/maven_test.go +++ b/internal/handler/maven_test.go @@ -31,15 +31,6 @@ func TestMavenParsePath(t *testing.T) { "short/path", "", "", "", "", }, - // Group directories never hold dots or are empty in the Maven - // layout; these would spell the same group as org/foo. - {"org.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, - {"org/.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, - {"org//foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, - // Cache identity is group:artifact, so a colon in either would - // let org:foo/bar and org/foo:bar share one entry. - {"org:foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, - {"org/foo:bar/1.0/bar-1.0.jar", "", "", "", ""}, } for _, tt := range tests { @@ -64,12 +55,8 @@ func TestMavenIsArtifactFile(t *testing.T) { {"guava-32.1.3-jre.module", true}, {"app-1.0.war", true}, {"lib-1.0.aar", true}, - {"apache-maven-3.9.9-bin.tar.gz", true}, - {"apache-maven-3.9.9-bin.zip", true}, - {"maven-mvnd-1.0.6-linux-amd64.tgz", true}, {"maven-metadata.xml", false}, {"guava-32.1.3-jre.jar.sha1", false}, - {"apache-maven-3.9.9-bin.tar.gz.sha512", false}, } for _, tt := range tests { @@ -80,74 +67,62 @@ func TestMavenIsArtifactFile(t *testing.T) { } } -func TestMavenIsVersionListingFile(t *testing.T) { +func TestMavenIsMetadataFile(t *testing.T) { h := &MavenHandler{} tests := []struct { + name string filename string want bool }{ - {"maven-metadata.xml", true}, - {"maven-metadata.xml.sha1", true}, - {"maven-metadata.xml.sha256", true}, - {"guava-32.1.3-jre.jar", false}, - {"guava-32.1.3-jre.jar.sha1", false}, - {"com.diffplug.spotless.gradle.plugin-8.4.0.pom", false}, + { + name: "pom is artifact, not metadata", + filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom", + want: false, + }, + { + name: "pom checksum is metadata", + filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1", + want: true, + }, + { + name: "metadata file", + filename: "maven-metadata.xml", + want: true, + }, + { + name: "metadata checksum", + filename: "maven-metadata.xml.sha256", + want: true, + }, + { + name: "jar checksum is metadata", + filename: "guava-32.1.3-jre.jar.sha1", + want: true, + }, + { + name: "asc signature is metadata", + filename: "guava-32.1.3-jre.jar.asc", + want: true, + }, + { + name: "regular jar is not metadata", + filename: "guava-32.1.3-jre.jar", + want: false, + }, + { + name: "pom checksum is metadata", + filename: "guava-32.1.3-jre.pom.sha1", + want: true, + }, } for _, tt := range tests { - if got := h.isVersionListingFile(tt.filename); got != tt.want { - t.Errorf("isVersionListingFile(%q) = %v, want %v", tt.filename, got, tt.want) - } - } -} - -func TestMavenIsChecksumFile(t *testing.T) { - h := &MavenHandler{} - - tests := []struct { - filename string - want bool - }{ - {"guava-32.1.3-jre.jar.sha1", true}, - {"guava-32.1.3-jre.jar.md5", true}, - {"guava-32.1.3-jre.pom.sha1", true}, - {"guava-32.1.3-jre.jar.asc", true}, - {"apache-maven-3.9.9-bin.tar.gz.sha512", true}, - {"com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1", true}, - {"guava-32.1.3-jre.jar", false}, - {"guava-32.1.3-jre.pom", false}, - {"maven-metadata.xml", false}, - } - - for _, tt := range tests { - if got := h.isChecksumFile(tt.filename); got != tt.want { - t.Errorf("isChecksumFile(%q) = %v, want %v", tt.filename, got, tt.want) - } - } -} - -func TestMavenIsCoordinateFile(t *testing.T) { - h := &MavenHandler{} - - tests := []struct { - path string - want bool - }{ - {"com/google/protobuf/protoc/3.21.0/protoc-3.21.0-linux-x86_64.exe", true}, - {"com/google/protobuf/protoc/3.21.0/protoc-3.21.0-linux-x86_64.exe.sha1", true}, - {"org/example/plugin/1.0/plugin-1.0.hpi", true}, - {"org/example/lib/1.0-SNAPSHOT/lib-1.0-20240101.120000-1.exe", true}, - {"org/example/lib/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.exe", true}, - {"com/google/protobuf/protoc/3.21.0/", false}, - {"com/google/protobuf/protoc/3.21.0/other-3.21.0.exe", false}, - {"com/google/protobuf/protoc/3.21.0/protoc-3.20.0.exe", false}, - {"archetype-catalog.xml", false}, - } - - for _, tt := range tests { - if got := h.isCoordinateFile(tt.path); got != tt.want { - t.Errorf("isCoordinateFile(%q) = %v, want %v", tt.path, got, tt.want) - } + t.Run(tt.name, func(t *testing.T) { + got := h.isMetadataFile(tt.filename) + if got != tt.want { + t.Errorf("isMetadataFile(%q) = %v, want %v", tt.filename, got, tt.want) + } + }) } } diff --git a/internal/handler/metadata_access_test.go b/internal/handler/metadata_access_test.go deleted file mode 100644 index 7b5e1cf..0000000 --- a/internal/handler/metadata_access_test.go +++ /dev/null @@ -1,181 +0,0 @@ -package handler - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/database" -) - -// ageMetadataAccess moves a cache row's last access an hour back and returns -// that time. -func ageMetadataAccess(t *testing.T, db *database.DB, ecosystem, key string) time.Time { - t.Helper() - old := time.Now().Add(-time.Hour).UTC() - query := db.Rebind(`UPDATE metadata_cache SET last_accessed_at = ? WHERE ecosystem = ? AND name = ?`) - if _, err := db.Exec(query, old, ecosystem, key); err != nil { - t.Fatalf("ageing last access: %v", err) - } - return old -} - -func assertMetadataAccessedAfter(t *testing.T, db *database.DB, ecosystem, key string, old time.Time) { - t.Helper() - entry, err := db.GetMetadataCache(ecosystem, key) - if err != nil || entry == nil { - t.Fatalf("GetMetadataCache: %v, %v", entry, err) - } - if !entry.LastAccessedAt.Valid || !entry.LastAccessedAt.Time.After(old.Add(time.Minute)) { - t.Errorf("last_accessed_at = %v, want the time it was served from cache", entry.LastAccessedAt) - } -} - -func TestFetchOrCacheMetadata_CacheHitMarksAccess(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"v":1}`)) - })) - t.Cleanup(upstream.Close) - - proxy, db, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - ctx := context.Background() - - if _, _, err := proxy.FetchOrCacheMetadata(ctx, "test", "pkg", upstream.URL+"/pkg"); err != nil { - t.Fatalf("first fetch: %v", err) - } - old := ageMetadataAccess(t, db, "test", "pkg") - if _, _, err := proxy.FetchOrCacheMetadata(ctx, "test", "pkg", upstream.URL+"/pkg"); err != nil { - t.Fatalf("cached fetch: %v", err) - } - assertMetadataAccessedAfter(t, db, "test", "pkg", old) -} - -func TestProxyCached_NotModifiedMarksAccess(t *testing.T) { - const body = `{"v":1}` - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("ETag", `"v1"`) - if r.Header.Get("If-None-Match") == `"v1"` { - w.WriteHeader(http.StatusNotModified) - return - } - _, _ = w.Write([]byte(body)) - })) - t.Cleanup(upstream.Close) - - proxy, db, store, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Nanosecond - proxy.HTTPClient = upstream.Client() - - for range 2 { - response := httptest.NewRecorder() - proxy.ProxyCached(response, httptest.NewRequest(http.MethodGet, "/pkg", nil), upstream.URL+"/pkg", "test", "pkg") - if response.Code != http.StatusOK || response.Body.String() != body { - t.Fatalf("response = %d %q, want 200 %q", response.Code, response.Body.String(), body) - } - if store.stores != 1 { - t.Fatalf("document stored %d times, want once", store.stores) - } - assertMetadataAccessedAfter(t, db, "test", "pkg", time.Now().Add(-time.Hour)) - ageMetadataAccess(t, db, "test", "pkg") - } -} - -func TestLoadContainerMetadata_MarksAccess(t *testing.T) { - proxy, db, _, _ := setupTestProxy(t) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", "") - ctx := context.Background() - - if _, err := h.storeContainerMetadata(ctx, "oci-test", "key", []byte("{}"), "", "", "application/json", "", time.Time{}, time.Now()); err != nil { - t.Fatalf("storing: %v", err) - } - old := ageMetadataAccess(t, db, "oci-test", "key") - if entry, _, err := h.loadContainerMetadata(ctx, "oci-test", "key"); err != nil || entry == nil { - t.Fatalf("loading: %v, %v", entry, err) - } - assertMetadataAccessedAfter(t, db, "oci-test", "key", old) -} - -func TestLoadContainerManifest_MarksAccess(t *testing.T) { - proxy, db, _, _ := setupTestProxy(t) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", "") - ctx := context.Background() - - manifest := &cachedContainerManifest{body: []byte("{}"), contentType: "application/json", fetchedAt: time.Now()} - if err := h.storeContainerManifest(ctx, "key", manifest); err != nil { - t.Fatalf("storing: %v", err) - } - old := ageMetadataAccess(t, db, containerManifestCacheEcosystem, "key") - if got, err := h.loadContainerManifest(ctx, "key"); err != nil || got == nil { - t.Fatalf("loading: %v, %v", got, err) - } - assertMetadataAccessedAfter(t, db, containerManifestCacheEcosystem, "key", old) -} - -func TestLoadContainerTags_MarksAccess(t *testing.T) { - proxy, db, _, _ := setupTestProxy(t) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", "") - ctx := context.Background() - - tags := &cachedContainerTags{body: []byte(`{"tags":[]}`), contentType: "application/json", fetchedAt: time.Now()} - if err := h.storeContainerTags(ctx, "key", tags); err != nil { - t.Fatalf("storing: %v", err) - } - old := ageMetadataAccess(t, db, containerTagsCacheEcosystem, "key") - if got, err := h.loadContainerTags(ctx, "key"); err != nil || got == nil { - t.Fatalf("loading: %v, %v", got, err) - } - assertMetadataAccessedAfter(t, db, containerTagsCacheEcosystem, "key", old) -} - -func TestStoredRewrite_MarksAccess(t *testing.T) { - proxy, db, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.SetMetadataRewriteCacheSize(1 << 20) - - body := []byte(`{}`) - proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) - key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { - t.Fatal(err) - } - old := ageMetadataAccess(t, db, "npm", "demo") - if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { - t.Fatal("expected a stored rewrite") - } - assertMetadataAccessedAfter(t, db, "npm", "demo", old) -} - -// TestArchHandler_RefreshRestoresEvictedBlob covers a database row whose blob -// size eviction removed while a refresh re-created the row: the next refresh -// with the same bytes stores the blob again, so the stale fallback works. -func TestArchHandler_RefreshRestoresEvictedBlob(t *testing.T) { - db := []byte("core-db-bytes") - upstream := newArchUpstream(t, map[string][]byte{"/core/os/x86_64/core.db": db}) - proxy := newArchTestProxy(t, upstream) - h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) - path := "/archlinux/core/os/x86_64/core.db" - - assertArchBody(t, serveArchRequest(h, http.MethodGet, path, nil), "seed", db) - entry, err := proxy.DB.GetMetadataCache(archEcosystem, h.metadataCacheKey("archlinux", upstream.URL, "core/os/x86_64/core.db")) - if err != nil || entry == nil { - t.Fatalf("metadata entry: %v, %v", entry, err) - } - if err := proxy.Storage.Delete(t.Context(), entry.StoragePath); err != nil { - t.Fatalf("deleting blob: %v", err) - } - - ageMetadata(t, proxy, h, "archlinux", upstream.URL, "core/os/x86_64/core.db") - assertArchBody(t, serveArchRequest(h, http.MethodGet, path, nil), "refresh", db) - - ageMetadata(t, proxy, h, "archlinux", upstream.URL, "core/os/x86_64/core.db") - upstream.available.Store(false) - assertArchBody(t, serveArchRequest(h, http.MethodGet, path, nil), "stale", db) -} diff --git a/internal/handler/metadata_coalesce_test.go b/internal/handler/metadata_coalesce_test.go deleted file mode 100644 index af8e2eb..0000000 --- a/internal/handler/metadata_coalesce_test.go +++ /dev/null @@ -1,249 +0,0 @@ -package handler - -import ( - "context" - "errors" - "net/http" - "net/http/httptest" - "sync" - "sync/atomic" - "testing" - "time" -) - -// metadataUpstream is a stub registry that counts metadata requests. It -// signals entered when the first request arrives and holds every request until -// release is closed, so a test can keep the first fetch in flight while others -// arrive. -type metadataUpstream struct { - *httptest.Server - calls atomic.Int64 - entered chan struct{} - release chan struct{} - once sync.Once -} - -func newMetadataUpstream(t *testing.T, status int) *metadataUpstream { - t.Helper() - u := &metadataUpstream{entered: make(chan struct{}), release: make(chan struct{})} - u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - u.calls.Add(1) - u.once.Do(func() { close(u.entered) }) - <-u.release - if status != http.StatusOK { - w.WriteHeader(status) - return - } - w.Header().Set("Content-Type", "application/json") - // Echo the Accept header so a test can tell which variant it got. - _, _ = w.Write([]byte(`{"accept":"` + r.Header.Get("Accept") + `"}`)) - })) - t.Cleanup(u.Close) - return u -} - -func metadataTestProxy(t *testing.T, u *metadataUpstream) *Proxy { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = u.Client() - return proxy -} - -// waitForMetadataWaiters returns once n callers are waiting on the fetch for -// key, so a test can release upstream knowing every caller joined it. -func waitForMetadataWaiters(t *testing.T, p *Proxy, key string, n int) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - p.metaMu.Lock() - f := p.inFlightMeta[key] - joined := 0 - if f != nil { - joined = f.waiters - } - p.metaMu.Unlock() - if joined >= n { - return - } - time.Sleep(time.Millisecond) - } - t.Fatalf("timed out waiting for %d callers to join the metadata fetch", n) -} - -type metadataCall struct { - body []byte - err error -} - -// startMetadataCalls starts n FetchOrCacheMetadata calls for one package. -func startMetadataCalls(p *Proxy, ctx context.Context, n int, url, accept string) []chan metadataCall { - out := make([]chan metadataCall, n) - for i := range out { - out[i] = make(chan metadataCall, 1) - go func(ch chan metadataCall) { - body, _, err := p.FetchOrCacheMetadata(ctx, "npm", "left-pad", url, accept) - ch <- metadataCall{body, err} - }(out[i]) - } - return out -} - -// TestFetchOrCacheMetadata_ConcurrentMissesCoalesce asserts that concurrent -// misses for one package make a single upstream request: the CI shape, where -// parallel jobs resolve the same Composer or npm dependencies at once. -func TestFetchOrCacheMetadata_ConcurrentMissesCoalesce(t *testing.T) { - u := newMetadataUpstream(t, http.StatusOK) - p := metadataTestProxy(t, u) - const n = 10 - - calls := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) - <-u.entered - calls = append(calls, startMetadataCalls(p, context.Background(), n-1, u.URL, contentTypeJSON)...) - key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) - waitForMetadataWaiters(t, p, key, n-1) - close(u.release) - - for _, ch := range calls { - c := <-ch - if c.err != nil { - t.Fatalf("FetchOrCacheMetadata: %v", c.err) - } - if string(c.body) != `{"accept":"application/json"}` { - t.Errorf("body = %s", c.body) - } - } - if got := u.calls.Load(); got != 1 { - t.Errorf("upstream requests = %d, want 1", got) - } -} - -// TestFetchOrCacheMetadata_DifferentAcceptDoNotShare asserts that requests -// for different variants of one package, such as npm's abbreviated and full -// documents, each get their own fetch and their own bytes. -func TestFetchOrCacheMetadata_DifferentAcceptDoNotShare(t *testing.T) { - u := newMetadataUpstream(t, http.StatusOK) - p := metadataTestProxy(t, u) - const abbreviated = "application/vnd.npm.install-v1+json" - - full := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) - <-u.entered - abbr := startMetadataCalls(p, context.Background(), 1, u.URL, abbreviated) - deadline := time.Now().Add(5 * time.Second) - for u.calls.Load() < 2 && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - close(u.release) - - if c := <-full[0]; c.err != nil || string(c.body) != `{"accept":"application/json"}` { - t.Errorf("full document: body %s, err %v", c.body, c.err) - } - if c := <-abbr[0]; c.err != nil || string(c.body) != `{"accept":"`+abbreviated+`"}` { - t.Errorf("abbreviated document: body %s, err %v", c.body, c.err) - } - if got := u.calls.Load(); got != 2 { - t.Errorf("upstream requests = %d, want 2", got) - } -} - -// TestFetchOrCacheMetadata_FirstCallerLeavingDoesNotFailOthers asserts that -// when the caller running the shared fetch disconnects, the fetch continues -// and everyone waiting on it still gets the metadata. -func TestFetchOrCacheMetadata_FirstCallerLeavingDoesNotFailOthers(t *testing.T) { - u := newMetadataUpstream(t, http.StatusOK) - p := metadataTestProxy(t, u) - const n = 5 - - leaderCtx, cancelLeader := context.WithCancel(context.Background()) - leader := startMetadataCalls(p, leaderCtx, 1, u.URL, contentTypeJSON) - <-u.entered - waiters := startMetadataCalls(p, context.Background(), n, u.URL, contentTypeJSON) - key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) - waitForMetadataWaiters(t, p, key, n) - cancelLeader() - close(u.release) - - <-leader[0] - for _, ch := range waiters { - if c := <-ch; c.err != nil { - t.Errorf("waiter failed after the first caller left: %v", c.err) - } - } - if got := u.calls.Load(); got != 1 { - t.Errorf("upstream requests = %d, want 1", got) - } -} - -// TestFetchOrCacheMetadata_WaiterLeavingKeepsFetch asserts that a waiter whose -// client disconnects returns its own context error without disturbing the -// fetch the others are waiting on. -func TestFetchOrCacheMetadata_WaiterLeavingKeepsFetch(t *testing.T) { - u := newMetadataUpstream(t, http.StatusOK) - p := metadataTestProxy(t, u) - - leader := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) - <-u.entered - waiterCtx, cancelWaiter := context.WithCancel(context.Background()) - leaving := startMetadataCalls(p, waiterCtx, 1, u.URL, contentTypeJSON) - key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) - waitForMetadataWaiters(t, p, key, 1) - cancelWaiter() - - if c := <-leaving[0]; !errors.Is(c.err, context.Canceled) { - t.Errorf("leaving waiter err = %v, want context.Canceled", c.err) - } - close(u.release) - if c := <-leader[0]; c.err != nil { - t.Errorf("first caller failed: %v", c.err) - } -} - -// TestFetchOrCacheMetadata_SharedNotFound asserts that an upstream 404 reaches -// every caller sharing the fetch as ErrUpstreamNotFound, from one request. -func TestFetchOrCacheMetadata_SharedNotFound(t *testing.T) { - u := newMetadataUpstream(t, http.StatusNotFound) - p := metadataTestProxy(t, u) - const n = 5 - - calls := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) - <-u.entered - calls = append(calls, startMetadataCalls(p, context.Background(), n-1, u.URL, contentTypeJSON)...) - key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) - waitForMetadataWaiters(t, p, key, n-1) - close(u.release) - - for _, ch := range calls { - if c := <-ch; !errors.Is(c.err, ErrUpstreamNotFound) { - t.Errorf("err = %v, want ErrUpstreamNotFound", c.err) - } - } - if got := u.calls.Load(); got != 1 { - t.Errorf("upstream requests = %d, want 1", got) - } -} - -// TestCoalescedMetadataMiss_ServesRowCommittedSinceLookup asserts that a -// caller which missed the cache, but finds the row fresh once it takes the -// key, serves that row instead of fetching again. That is a fetch finishing -// between a caller's lookup and its turn at the key. -func TestCoalescedMetadataMiss_ServesRowCommittedSinceLookup(t *testing.T) { - u := newMetadataUpstream(t, http.StatusOK) - close(u.release) - p := metadataTestProxy(t, u) - p.CacheMetadata = true - p.MetadataTTL = time.Hour - - if _, _, err := p.FetchOrCacheMetadata(context.Background(), "npm", "left-pad", u.URL); err != nil { - t.Fatalf("priming fetch: %v", err) - } - - res := p.coalescedMetadataMiss(context.Background(), "npm", "left-pad", u.URL, contentTypeJSON, "", nil) - if res.err != nil { - t.Fatalf("coalescedMetadataMiss: %v", res.err) - } - if string(res.body) != `{"accept":"application/json"}` { - t.Errorf("body = %s", res.body) - } - if got := u.calls.Load(); got != 1 { - t.Errorf("upstream requests = %d, want 1 (the priming fetch only)", got) - } -} diff --git a/internal/handler/metadata_dates.go b/internal/handler/metadata_dates.go deleted file mode 100644 index fca5a1f..0000000 --- a/internal/handler/metadata_dates.go +++ /dev/null @@ -1,83 +0,0 @@ -package handler - -import ( - "time" - - "github.com/git-pkgs/proxy/internal/database" -) - -// metadataDateAttempts bounds how often metadataDate retries when other -// replicas keep changing a package's date between its read and its write. -const metadataDateAttempts = 3 - -// metadataDate returns the Last-Modified to serve a package's metadata with, -// as rewritten for proxyURL into the representation with the given ETag, or -// zero for none. -// -// The date is the second the package started serving that representation, -// recorded in the database so that every replica sharing it serves the same -// date. A switch to another representation is dated only in a second later -// than the recorded date, and then with that second, so a date is never given -// to two representations of a package: a client holding one never gets 304 -// for another, whichever replica it asks. Until such a second the -// representation goes undated, and If-Modified-Since is not evaluated. -// -// A package's first date is likewise later than the second this process -// started, as the process it replaced may have dated the package then without -// recording it. A date ahead of this process's clock, recorded by a replica -// whose clock is ahead, is not served until the clock catches up. Without a -// database, or when it fails, nothing is dated. -func (p *Proxy) metadataDate(ecosystem, proxyURL, name, etag string) time.Time { - if p.DB == nil || etag == "" { - return time.Time{} - } - now := p.clock().Truncate(time.Second) - for range metadataDateAttempts { - date, settled, err := p.claimMetadataDate(ecosystem, proxyURL, name, etag, now) - if err != nil { - p.Logger.Warn("failed to date rewritten metadata", "ecosystem", ecosystem, "name", name, "error", err) - return time.Time{} - } - if settled { - if date.After(now) { - return time.Time{} - } - return date - } - } - return time.Time{} -} - -// claimMetadataDate makes one attempt at recording the date of etag for a -// package at now. It reports false when another replica changed the date -// first, and the attempt should be made again. -func (p *Proxy) claimMetadataDate(ecosystem, proxyURL, name, etag string, now time.Time) (time.Time, bool, error) { - current, err := p.DB.GetMetadataDate(ecosystem, proxyURL, name) - if err != nil { - return time.Time{}, false, err - } - next := database.MetadataDate{ETag: etag, LastModified: now} - switch { - case current == nil: - if !now.After(p.started.Truncate(time.Second)) { - return time.Time{}, true, nil - } - inserted, err := p.DB.InsertMetadataDate(ecosystem, proxyURL, name, next) - return now, inserted, err - case current.ETag == etag: - return current.LastModified, true, nil - case !now.After(current.LastModified): - return time.Time{}, true, nil - default: - swapped, err := p.DB.SwapMetadataDate(ecosystem, proxyURL, name, *current, next) - return now, swapped, err - } -} - -// clock reads the time metadata is dated by. -func (p *Proxy) clock() time.Time { - if p.now == nil { - return time.Now() - } - return p.now() -} diff --git a/internal/handler/metadata_dates_upgrade_test.go b/internal/handler/metadata_dates_upgrade_test.go deleted file mode 100644 index 4d1ca74..0000000 --- a/internal/handler/metadata_dates_upgrade_test.go +++ /dev/null @@ -1,234 +0,0 @@ -package handler - -import ( - "context" - "crypto/rand" - "encoding/hex" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "net/url" - "os" - "strings" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/registries/fetch" -) - -// upgradeDatabases opens an empty database with the current schema for each -// dialect the proxy supports. A nil database means the dialect is unavailable. -var upgradeDatabases = []struct { - dialect string - open func(t *testing.T) *database.DB -}{ - {"sqlite", func(t *testing.T) *database.DB { - db, err := database.Create(t.TempDir() + "/test.db") - if err != nil { - t.Fatalf("Create failed: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - return db - }}, - {"postgres", openTestPostgres}, -} - -// openTestPostgres creates a database of its own on the server at -// PROXY_DATABASE_URL, so the test shares no tables with tests elsewhere that -// use the same server, and drops it afterwards. -func openTestPostgres(t *testing.T) *database.DB { - t.Helper() - serverURL := os.Getenv("PROXY_DATABASE_URL") - if serverURL == "" { - return nil - } - server, err := database.OpenPostgres(serverURL) - if err != nil { - t.Fatalf("OpenPostgres failed: %v", err) - } - t.Cleanup(func() { _ = server.Close() }) - - suffix := make([]byte, 8) - _, _ = rand.Read(suffix) - name := "proxy_upgrade_" + hex.EncodeToString(suffix) - if _, err := server.Exec("CREATE DATABASE " + name); err != nil { - t.Fatalf("creating database %s: %v", name, err) - } - t.Cleanup(func() { _, _ = server.Exec("DROP DATABASE IF EXISTS " + name + " WITH (FORCE)") }) - - dbURL, err := url.Parse(serverURL) - if err != nil { - t.Fatal(err) - } - dbURL.Path = "/" + name - db, err := database.OpenPostgres(dbURL.String()) - if err != nil { - t.Fatalf("OpenPostgres(%s) failed: %v", name, err) - } - t.Cleanup(func() { _ = db.Close() }) - if err := db.CreateSchema(); err != nil { - t.Fatalf("CreateSchema failed: %v", err) - } - return db -} - -// rollBackMetadataDates returns db to the schema from before migration 011, -// which only added the metadata_dates table. -func rollBackMetadataDates(t *testing.T, db *database.DB) { - t.Helper() - if _, err := db.Exec("DROP TABLE metadata_dates"); err != nil { - t.Fatal(err) - } - if _, err := db.Exec("DELETE FROM migrations WHERE name = '011_add_metadata_dates'"); err != nil { - t.Fatal(err) - } -} - -// upgradeProxy returns a proxy on db that dates metadata by a clock the test -// moves, a second after the process start it models. -func upgradeProxy(t *testing.T, db *database.DB, u *validatorUpstream) (*Proxy, *testClock) { - t.Helper() - proxy := NewProxy(db, newMockStorage(), &mockFetcher{}, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil))) - proxy.HTTPClient = u.Client() - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} - restartProxy(proxy, clock) - clock.advance(time.Second) - return proxy, clock -} - -// cachedMetadata is a metadata cache row and the bytes stored for it. -type cachedMetadata struct { - entry database.MetadataCacheEntry - body string -} - -func readCachedMetadata(t *testing.T, proxy *Proxy, ecosystem, name string) cachedMetadata { - t.Helper() - entry, err := proxy.DB.GetMetadataCache(ecosystem, name) - if err != nil || entry == nil { - t.Fatalf("metadata row %s/%s = %v, %v", ecosystem, name, entry, err) - } - r, err := proxy.Storage.Open(context.Background(), entry.StoragePath) - if err != nil { - t.Fatalf("opening stored metadata %s: %v", entry.StoragePath, err) - } - defer func() { _ = r.Close() }() - body, err := io.ReadAll(r) - if err != nil { - t.Fatal(err) - } - return cachedMetadata{entry: *entry, body: string(body)} -} - -// TestRewrittenMetadataAfterMetadataDatesMigration upgrades a database from -// before migration 011 that holds cached npm and Composer metadata. The cached -// rows and stored bytes must come through unchanged, and two replicas sharing -// the upgraded database must serve the cached document with validators that -// revalidate, then date a changed document alike. -func TestRewrittenMetadataAfterMetadataDatesMigration(t *testing.T) { - for _, d := range upgradeDatabases { - for _, c := range validatorCases { - t.Run(d.dialect+"/"+c.ecosystem, func(t *testing.T) { - db := d.open(t) - if db == nil { - t.Skip("PROXY_DATABASE_URL not set, skipping postgres test") - } - u := newValidatorUpstream(t) - a, clock := upgradeProxy(t, db, u) - upgradeWithCachedMetadata(t, a, u, c) - - replicas := map[string]http.Handler{"A": c.handler(a, u), "B": c.handler(validatorReplica(a, clock), u)} - cached := serve(replicas["A"], c.path, nil) - if cached.Code != http.StatusOK || strings.Contains(cached.Body.String(), "2.0.0") { - t.Fatalf("first request after the upgrade: status %d, body %s; want 200 with the document cached before it", cached.Code, cached.Body) - } - expectRevalidates(t, replicas, c.path, cached) - - expireMetadata(t, db, c.ecosystem, c.name) - clock.advance(time.Second) - changed := serve(replicas["A"], c.path, nil) - expectChangedDocument(t, cached, changed) - expectChangedForCachedValidators(t, replicas, c.path, cached, changed) - expectRevalidates(t, replicas, c.path, changed) - }) - } - } -} - -// upgradeWithCachedMetadata rolls proxy's database back to before migration -// 011, caches c's document as the proxy did then, moves upstream on to version -// 2.0.0 and migrates. The cached row and stored bytes must survive. -func upgradeWithCachedMetadata(t *testing.T, proxy *Proxy, u *validatorUpstream, c validatorCase) { - t.Helper() - rollBackMetadataDates(t, proxy.DB) - if _, _, err := proxy.FetchOrCacheMetadata(context.Background(), c.ecosystem, c.name, u.URL+c.path); err != nil { - t.Fatalf("seeding metadata before the upgrade: %v", err) - } - seeded := readCachedMetadata(t, proxy, c.ecosystem, c.name) - u.set(npmV("2.0.0"), composerV("2.0.0")) - - if err := proxy.DB.MigrateSchema(); err != nil { - t.Fatalf("MigrateSchema() error = %v", err) - } - if upgraded := readCachedMetadata(t, proxy, c.ecosystem, c.name); upgraded != seeded { - t.Fatalf("cached metadata after the upgrade = %+v, want %+v", upgraded, seeded) - } -} - -// expectChangedDocument checks that changed is the new upstream document, -// with an ETag of its own and a later date than cached. -func expectChangedDocument(t *testing.T, cached, changed *httptest.ResponseRecorder) { - t.Helper() - if changed.Code != http.StatusOK || !strings.Contains(changed.Body.String(), "2.0.0") { - t.Fatalf("after upstream changed: status %d, body %s; want 200 with the new document", changed.Code, changed.Body) - } - if changed.Header().Get("ETag") == cached.Header().Get("ETag") { - t.Error("the changed document has the cached document's ETag") - } - was, _ := http.ParseTime(cached.Header().Get("Last-Modified")) - if now, err := http.ParseTime(changed.Header().Get("Last-Modified")); err != nil || !now.After(was) { - t.Errorf("changed document Last-Modified %q, want later than %q", changed.Header().Get("Last-Modified"), cached.Header().Get("Last-Modified")) - } -} - -// expectChangedForCachedValidators checks that no replica answers a client -// holding the cached document's validators with 304 once it changed. -func expectChangedForCachedValidators(t *testing.T, replicas map[string]http.Handler, path string, cached, changed *httptest.ResponseRecorder) { - t.Helper() - held := []http.Header{ - {"If-None-Match": {cached.Header().Get("ETag")}}, - {"If-Modified-Since": {cached.Header().Get("Last-Modified")}}, - } - for name, handler := range replicas { - for _, header := range held { - if rec := serve(handler, path, header); rec.Code != http.StatusOK || rec.Body.String() != changed.Body.String() { - t.Errorf("replica %s, %v of the cached document: status %d; want 200 with the changed document", name, header, rec.Code) - } - } - } -} - -// expectRevalidates checks that every replica serves want's validators and -// answers a conditional request with either of them with 304. -func expectRevalidates(t *testing.T, replicas map[string]http.Handler, path string, want *httptest.ResponseRecorder) { - t.Helper() - etag, lastModified := want.Header().Get("ETag"), want.Header().Get("Last-Modified") - if etag == "" || lastModified == "" { - t.Fatalf("ETag %q, Last-Modified %q; want both", etag, lastModified) - } - for name, handler := range replicas { - rec := serve(handler, path, nil) - if rec.Header().Get("ETag") != etag || rec.Header().Get("Last-Modified") != lastModified { - t.Errorf("replica %s: ETag %q, Last-Modified %q; want %q, %q", name, rec.Header().Get("ETag"), rec.Header().Get("Last-Modified"), etag, lastModified) - } - for _, header := range []http.Header{{"If-None-Match": {etag}}, {"If-Modified-Since": {lastModified}}} { - if rec := serve(handler, path, header); rec.Code != http.StatusNotModified { - t.Errorf("replica %s, %v: status %d, want 304", name, header, rec.Code) - } - } - } -} diff --git a/internal/handler/metadata_encoding_test.go b/internal/handler/metadata_encoding_test.go deleted file mode 100644 index 10aab10..0000000 --- a/internal/handler/metadata_encoding_test.go +++ /dev/null @@ -1,200 +0,0 @@ -package handler - -import ( - "bytes" - "compress/gzip" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" - "time" -) - -// gzipPayload returns a gzip-compressed copy of data, simulating an origin -// that stores pre-compressed index files. -func gzipPayload(t *testing.T, data []byte) []byte { - t.Helper() - var buf bytes.Buffer - zw := gzip.NewWriter(&buf) - if _, err := zw.Write(data); err != nil { - t.Fatalf("compressing payload: %v", err) - } - if err := zw.Close(); err != nil { - t.Fatalf("closing gzip writer: %v", err) - } - return buf.Bytes() -} - -// TestProxyCached_PreservesContentEncodedBytes covers issue #300: an upstream -// that serves a signed index with Content-Encoding: gzip must have its bytes -// cached and re-served verbatim, with the encoding header replayed, instead of -// being transparently decompressed by the HTTP client. -func TestProxyCached_PreservesContentEncodedBytes(t *testing.T) { - raw := gzipPayload(t, []byte("signed index payload")) - - var available atomic.Bool - available.Store(true) - var sawAcceptEncoding atomic.Value - var upstreamRequests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - upstreamRequests.Add(1) - sawAcceptEncoding.Store(r.Header.Get(headerAcceptEncoding)) - w.Header().Set(headerContentType, "application/octet-stream") - w.Header().Set(headerContentEncoding, "gzip") - _, _ = w.Write(raw) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - - serve := func() *httptest.ResponseRecorder { - w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodGet, "/index", nil) - proxy.ProxyCached(w, r, upstream.URL+"/index", "apk", "index-key", "*/*") - return w - } - - first := serve() - if first.Code != http.StatusOK { - t.Fatalf("first response status = %d, want 200: %s", first.Code, first.Body.String()) - } - if got, _ := sawAcceptEncoding.Load().(string); got != "identity" { - t.Errorf("upstream saw Accept-Encoding %q, want %q", got, "identity") - } - if !bytes.Equal(first.Body.Bytes(), raw) { - t.Errorf("first response altered the upstream bytes: got %d bytes, want %d", first.Body.Len(), len(raw)) - } - if got := first.Header().Get(headerContentEncoding); got != "gzip" { - t.Errorf("first response Content-Encoding = %q, want %q", got, "gzip") - } - - // Within the TTL and with the upstream down, the cached copy must be - // served with the same bytes and encoding. - available.Store(false) - second := serve() - if second.Code != http.StatusOK { - t.Fatalf("cached response status = %d, want 200: %s", second.Code, second.Body.String()) - } - if !bytes.Equal(second.Body.Bytes(), raw) { - t.Errorf("cached response altered the stored bytes") - } - if got := second.Header().Get(headerContentEncoding); got != "gzip" { - t.Errorf("cached response Content-Encoding = %q, want %q", got, "gzip") - } - if got := upstreamRequests.Load(); got != 1 { - t.Errorf("upstream requests = %d, want 1", got) - } -} - -// TestProxyCached_NoEncodingHeaderForIdentityResponses pins that ordinary -// responses do not grow a spurious Content-Encoding header. -func TestProxyCached_NoEncodingHeaderForIdentityResponses(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set(headerContentType, contentTypeJSON) - _, _ = w.Write([]byte(`{"ok":true}`)) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - - w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodGet, "/meta", nil) - proxy.ProxyCached(w, r, upstream.URL+"/meta", "npm", "meta-key", contentTypeJSON) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - if got := w.Header().Get(headerContentEncoding); got != "" { - t.Errorf("Content-Encoding = %q, want empty", got) - } -} - -// TestProxyMetadataStream_PreservesSignedBytesWithoutClientEncoding pins the -// uncached streaming path (the default, since cache_metadata is off) for the -// realistic client that sends no Accept-Encoding: the proxy must request -// identity upstream so Go does not transparently decompress a signed index, -// and the raw bytes plus the Content-Encoding header must reach the client. -func TestProxyMetadataStream_PreservesSignedBytesWithoutClientEncoding(t *testing.T) { - raw := gzipPayload(t, []byte("streamed index payload")) - var sawAcceptEncoding string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - sawAcceptEncoding = r.Header.Get(headerAcceptEncoding) - w.Header().Set(headerContentType, "application/octet-stream") - w.Header().Set(headerContentEncoding, "gzip") - _, _ = w.Write(raw) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = false - proxy.HTTPClient = upstream.Client() - - w := httptest.NewRecorder() - // No Accept-Encoding on the client request -- the apk/apt/dnf case. - r := httptest.NewRequest(http.MethodGet, "/index", nil) - proxy.ProxyCached(w, r, upstream.URL+"/index", "apk", "stream-key", "*/*") - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - if sawAcceptEncoding != "identity" { - t.Errorf("upstream saw Accept-Encoding %q, want %q", sawAcceptEncoding, "identity") - } - if !bytes.Equal(w.Body.Bytes(), raw) { - t.Errorf("streamed response altered the upstream bytes: got %d bytes, want %d", w.Body.Len(), len(raw)) - } - if got := w.Header().Get(headerContentEncoding); got != "gzip" { - t.Errorf("Content-Encoding = %q, want %q", got, "gzip") - } -} - -// TestFetchOrCacheMetadata_DirectCallersKeepTransparentCompression pins that -// the parsing/rewriting ecosystems (npm, pypi, cargo, helm, ...) that call -// FetchOrCacheMetadata directly are NOT forced to identity: they keep Go's -// transparent transfer compression and receive decoded bytes, so a gzip-only -// upstream does not regress them (no wire-size blowup, no parse failures). -func TestFetchOrCacheMetadata_DirectCallersKeepTransparentCompression(t *testing.T) { - plaintext := []byte(`{"name":"demo","versions":{"1.0.0":{}}}`) - var sawAcceptEncoding string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - sawAcceptEncoding = r.Header.Get(headerAcceptEncoding) - // Serve gzip only when the client accepts it, like a real CDN. - if strings.Contains(r.Header.Get(headerAcceptEncoding), "gzip") { - w.Header().Set(headerContentType, contentTypeJSON) - w.Header().Set(headerContentEncoding, "gzip") - _, _ = w.Write(gzipPayload(t, plaintext)) - return - } - w.Header().Set(headerContentType, contentTypeJSON) - _, _ = w.Write(plaintext) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - - body, _, err := proxy.FetchOrCacheMetadata(t.Context(), "npm", "demo", upstream.URL+"/demo", contentTypeJSON) - if err != nil { - t.Fatalf("FetchOrCacheMetadata() error = %v", err) - } - // The default transport adds Accept-Encoding: gzip and transparently - // decompresses, so the caller sees decoded JSON regardless of the wire form. - if sawAcceptEncoding == "identity" { - t.Errorf("direct caller forced identity; want transparent compression") - } - if !bytes.Equal(body, plaintext) { - t.Errorf("direct caller got %q, want decoded %q", body, plaintext) - } -} diff --git a/internal/handler/metadata_gzip_test.go b/internal/handler/metadata_gzip_test.go deleted file mode 100644 index 57d20cc..0000000 --- a/internal/handler/metadata_gzip_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package handler - -import ( - "bytes" - "compress/gzip" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// compressibleDocuments sets upstream documents with enough versions that -// gzip makes them smaller. -func compressibleDocuments(u *validatorUpstream) { - var npm, composer []string - for i := range 50 { - v := fmt.Sprintf("1.0.%d", i) - npm = append(npm, `"`+v+`":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-`+v+`.tgz"}}`) - composer = append(composer, `{"version":"`+v+`","dist":{"type":"zip","url":"https://example.com/pkg-`+v+`.zip"}}`) - } - u.set( - `{"name":"left-pad","versions":{`+strings.Join(npm, ",")+`}}`, - `{"packages":{"vendor/pkg":[`+strings.Join(composer, ",")+`]}}`, - ) -} - -func gunzip(t *testing.T, rec *httptest.ResponseRecorder) string { - t.Helper() - zr, err := gzip.NewReader(bytes.NewReader(rec.Body.Bytes())) - if err != nil { - t.Fatalf("body is not gzip: %v", err) - } - body, err := io.ReadAll(zr) - if err != nil { - t.Fatalf("reading gzip body: %v", err) - } - return string(body) -} - -var acceptGzip = http.Header{"Accept-Encoding": {"gzip, deflate, br"}} - -// TestRewrittenMetadataGzip checks that a client accepting gzip gets the -// cached compressed copy of rewritten metadata, on the first fetch and on -// the rewrite fast path, with an ETag of its own so it never matches the -// uncompressed representation. -func TestRewrittenMetadataGzip(t *testing.T) { - for _, c := range validatorCases { - for _, ttl := range []time.Duration{time.Hour, 0} { - t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { - u := newValidatorUpstream(t) - compressibleDocuments(u) - handler := c.handler(validatorProxy(t, u, ttl), u) - - identity := serve(handler, c.path, nil) - for range 2 { - expectGzipOf(t, serve(handler, c.path, acceptGzip), identity) - } - }) - } - } -} - -// expectGzipOf checks that rec is the gzip representation of the -// uncompressed response identity. -func expectGzipOf(t *testing.T, rec, identity *httptest.ResponseRecorder) { - t.Helper() - if rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "gzip" { - t.Fatalf("status %d, Content-Encoding %q; want 200 with gzip", rec.Code, rec.Header().Get("Content-Encoding")) - } - if rec.Body.Len() >= identity.Body.Len() { - t.Errorf("gzip body is %d bytes, uncompressed %d", rec.Body.Len(), identity.Body.Len()) - } - if got := gunzip(t, rec); got != identity.Body.String() { - t.Errorf("decompressed body differs from the uncompressed response") - } - if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { - t.Errorf("Vary = %q, want Accept-Encoding", got) - } - if etag := rec.Header().Get("ETag"); etag == "" || etag == identity.Header().Get("ETag") { - t.Errorf("gzip ETag %q must differ from the uncompressed ETag %q", etag, identity.Header().Get("ETag")) - } - if rec.Header().Get("Last-Modified") != identity.Header().Get("Last-Modified") { - t.Errorf("Last-Modified %q, want %q", rec.Header().Get("Last-Modified"), identity.Header().Get("Last-Modified")) - } -} - -// TestRewrittenMetadataGzipRevalidation checks that each representation -// revalidates with its own ETag, also when a reverse proxy weakened it. -func TestRewrittenMetadataGzipRevalidation(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := newValidatorUpstream(t) - compressibleDocuments(u) - handler := c.handler(validatorProxy(t, u, time.Hour), u) - gzipETag := serve(handler, c.path, acceptGzip).Header().Get("ETag") - identityETag := serve(handler, c.path, nil).Header().Get("ETag") - - for _, ifNoneMatch := range []string{gzipETag, "W/" + gzipETag} { - rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}, "If-None-Match": {ifNoneMatch}}) - expectNotModified(t, rec, gzipETag) - if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { - t.Errorf("304 Vary = %q, want Accept-Encoding", got) - } - } - if rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}, "If-None-Match": {identityETag}}); rec.Code != http.StatusOK { - t.Errorf("gzip request revalidating the uncompressed ETag: status %d, want 200", rec.Code) - } - if rec := serve(handler, c.path, http.Header{"If-None-Match": {gzipETag}}); rec.Code != http.StatusOK { - t.Errorf("uncompressed request revalidating the gzip ETag: status %d, want 200", rec.Code) - } - }) - } -} - -// TestRewrittenMetadataWithoutGzip checks that clients that do not accept -// gzip get the uncompressed document, which still varies on Accept-Encoding. -func TestRewrittenMetadataWithoutGzip(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := newValidatorUpstream(t) - compressibleDocuments(u) - handler := c.handler(validatorProxy(t, u, time.Hour), u) - - for _, acceptEncoding := range []string{"", "identity", "gzip;q=0", "br"} { - header := http.Header{} - if acceptEncoding != "" { - header.Set("Accept-Encoding", acceptEncoding) - } - rec := serve(handler, c.path, header) - if rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "" { - t.Errorf("Accept-Encoding %q: status %d, Content-Encoding %q; want 200 uncompressed", acceptEncoding, rec.Code, rec.Header().Get("Content-Encoding")) - } - if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { - t.Errorf("Accept-Encoding %q: Vary = %q, want Accept-Encoding", acceptEncoding, got) - } - } - }) - } -} - -// TestRewrittenMetadataGzipOnlyFromRewriteCache checks that rewrites made -// outside the cache, with it disabled or with cooldown on, stay uncompressed: -// compressing them on every request is better left to a reverse proxy. -func TestRewrittenMetadataGzipOnlyFromRewriteCache(t *testing.T) { - setups := map[string]func(*Proxy){ - "cache disabled": func(p *Proxy) { p.SetMetadataRewriteCacheSize(0) }, - "cooldown without cache": func(p *Proxy) { - p.SetMetadataRewriteCacheSize(0) - p.Cooldown = &cooldown.Config{Default: "3d"} - }, - } - for _, c := range validatorCases { - for name, setup := range setups { - t.Run(c.ecosystem+"/"+name, func(t *testing.T) { - u := newValidatorUpstream(t) - compressibleDocuments(u) - proxy := validatorProxy(t, u, time.Hour) - setup(proxy) - handler := c.handler(proxy, u) - - for range 2 { - if rec := serve(handler, c.path, acceptGzip); rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "" { - t.Errorf("status %d, Content-Encoding %q; want 200 uncompressed", rec.Code, rec.Header().Get("Content-Encoding")) - } - } - }) - } - } -} - -// TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller checks that the cache keeps -// only a compressed copy of a rewrite, counted against its size, unless -// compressing does not make the rewrite smaller, and that the plain document -// is still there for callers that need it. -func TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller(t *testing.T) { - c := newRewriteCache(1 << 20) - identity := func(b []byte) ([]byte, error) { return b, nil } - large := []byte(strings.Repeat(`{"version":"1.0.0"},`, 100)) - - for _, doc := range [][]byte{large, []byte("abc")} { - key := rewriteCacheKey("npm", "http://proxy", string(doc[:3]), doc) - if _, err := c.rewrite(t.Context(), key, doc, nil, identity); err != nil { - t.Fatal(err) - } - } - - m, ok := c.get(rewriteCacheKey("npm", "http://proxy", string(large[:3]), large)) - if !ok || m.gzipped == nil { - t.Fatalf("large rewrite: cached %v, gzip copy %d bytes; want a gzip copy", ok, len(m.gzipped)) - } - if m.plain != nil { - t.Errorf("large rewrite: cached %d plain bytes besides the gzip copy, want none", len(m.plain)) - } - if c.size != int64(len(m.gzipped)+3) { - t.Errorf("cache size = %d, want the gzip copy and the small rewrite (%d)", c.size, len(m.gzipped)+3) - } - if !bytes.Equal(m.content(), large) || m.size != len(large) { - t.Errorf("large rewrite content = %d bytes, size %d; want the %d bytes rewritten", len(m.content()), m.size, len(large)) - } - if small, ok := c.get(rewriteCacheKey("npm", "http://proxy", "abc", []byte("abc"))); !ok || small.gzipped != nil || string(small.content()) != "abc" { - t.Errorf("small rewrite: cached %v, gzip copy %d bytes, content %q; want abc uncompressed", ok, len(small.gzipped), small.content()) - } -} - -func TestAcceptsGzip(t *testing.T) { - cases := map[string]bool{ - "": false, - "identity": false, - "br, deflate": false, - "gzip": true, - "GZIP": true, - "x-gzip": true, - "deflate, gzip;q=1.0": true, - "gzip; q=0.5": true, - "gzip;q=0": false, - "gzip;q=0.000": false, - "*": true, - "*;q=0": false, - "br;q=1, *;q=0.5": true, - "gzip;q=0, *": false, - "gzip;q=invalid, deflate": false, - } - for header, want := range cases { - r := httptest.NewRequest(http.MethodGet, "/", nil) - if header != "" { - r.Header.Set("Accept-Encoding", header) - } - if got := acceptsGzip(r); got != want { - t.Errorf("acceptsGzip(%q) = %v, want %v", header, got, want) - } - } -} diff --git a/internal/handler/metadata_immutable.go b/internal/handler/metadata_immutable.go deleted file mode 100644 index 691fbc1..0000000 --- a/internal/handler/metadata_immutable.go +++ /dev/null @@ -1,116 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "net/http" - - "github.com/git-pkgs/proxy/internal/metrics" -) - -// Some metadata documents never change once published: a Go module's .mod -// and .info for a canonical version, or an RPM repodata file named after its -// own checksum. Revalidating them every metadata TTL, or fetching them again -// on every request without cache_metadata, only costs upstream requests. They -// are kept in the metadata cache like other metadata, but served whatever -// their age and stored whether or not cache_metadata is set, the way -// artifacts are. - -// errImmutableDigest is returned when a stored immutable document doesn't -// match the digest recorded for it. -var errImmutableDigest = errors.New("stored metadata does not match its digest") - -// proxyImmutable answers a request for an immutable metadata document: from -// the metadata cache when it is stored, otherwise fetched once and stored. A -// document over the metadata size limit is streamed from upstream instead, -// uncached. -func (p *Proxy) proxyImmutable(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, accept string) { - if p.serveStoredMetadata(w, r, ecosystem, cacheKey, true) { - return - } - res := p.fetchImmutableMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, accept) - if errors.Is(res.err, ErrMetadataTooLarge) { - p.proxyMetadataStream(w, r, upstreamURL, "identity", accept) - return - } - if res.err != nil { - p.writeMetadataFetchError(w, res.err) - return - } - cm := p.lookupCachedMeta(ecosystem, cacheKey) - cm.stale = false - writeMetadataBody(w, r, cm, res.body, res.contentType, res.contentEncoding) -} - -// fetchImmutableMetadata returns an immutable metadata document from the -// metadata cache whatever its age, or fetches and stores it. Upstream's bytes -// are kept as sent (Accept-Encoding: identity). Without a database or storage -// it falls back to the TTL lookup. -func (p *Proxy) fetchImmutableMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string) metadataResult { - if p.DB == nil || p.Storage == nil { - return p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, "identity", nil) - } - if containsPathTraversal(cacheKey) { - return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} - } - if hit := p.storedImmutable(ctx, ecosystem, cacheKey); hit != nil { - metrics.RecordMetadataCacheHit(ecosystem, len(hit.body)) - return *hit - } - metrics.RecordMetadataCacheMiss(ecosystem) - - // Kept apart from TTL lookups of the same key, which don't store the - // document without cache_metadata. - key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, "identity", false) + "\x00immutable" - res, _ := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { - if hit := p.storedImmutable(fetchCtx, ecosystem, cacheKey); hit != nil { - return *hit - } - meta, err := p.fetchUpstreamMetadata(fetchCtx, upstreamURL, nil, accept, "identity") - if err != nil { - return metadataResult{err: err} - } - digest := p.cacheMetadataBlob(fetchCtx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} - }) - return res -} - -// storedImmutable reads an immutable document from the metadata cache, or -// returns nil. Since the copy is never revalidated, it is checked against the -// digest recorded when it was stored; a row without one, or whose bytes don't -// match, is a miss, and the fetch that follows replaces it. -func (p *Proxy) storedImmutable(ctx context.Context, ecosystem, cacheKey string) *metadataResult { - entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) - if err != nil || entry == nil { - return nil - } - entry = p.migrateLegacyMetadataEntry(ctx, ecosystem, cacheKey, entry) - if entry == nil || !entry.ContentDigest.Valid { - return nil - } - want := entry.ContentDigest.String - body, contentType, err := p.readCachedMetadata(ctx, entry, func(b []byte) error { - sum := sha256.Sum256(b) - if "sha256:"+hex.EncodeToString(sum[:]) != want { - return errImmutableDigest - } - return nil - }) - if errors.Is(err, errImmutableDigest) { - p.Logger.Error("cached metadata failed integrity check", - "ecosystem", ecosystem, "key", cacheKey, "path", entry.StoragePath) - metrics.RecordIntegrityFailure(ecosystem, metrics.FailureDetail{Path: entry.StoragePath, Err: err}) - if err := p.DB.DiscardMetadataCache(ecosystem, cacheKey, entry.StoragePath, want); err != nil { - p.Logger.Warn("failed to clear corrupt metadata from cache", "error", err) - } - } - if err != nil { - return nil - } - res := metadataResult{body: body, contentType: contentType, contentEncoding: entry.ContentEncoding.String}.describedBy(entry) - return &res -} diff --git a/internal/handler/metadata_immutable_test.go b/internal/handler/metadata_immutable_test.go deleted file mode 100644 index 95b1a17..0000000 --- a/internal/handler/metadata_immutable_test.go +++ /dev/null @@ -1,270 +0,0 @@ -package handler - -import ( - "context" - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// countingUpstream answers every path with body and counts requests per path. -type countingUpstream struct { - server *httptest.Server - mu sync.Mutex - hits map[string]int - body func(path string) (string, bool) -} - -func newCountingUpstream(t *testing.T, body func(path string) (string, bool)) *countingUpstream { - t.Helper() - u := &countingUpstream{hits: map[string]int{}, body: body} - u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - u.mu.Lock() - u.hits[r.URL.Path]++ - u.mu.Unlock() - b, ok := u.body(r.URL.Path) - if !ok { - http.NotFound(w, r) - return - } - _, _ = w.Write([]byte(b)) - })) - t.Cleanup(u.server.Close) - return u -} - -func (u *countingUpstream) count(path string) int { - u.mu.Lock() - defer u.mu.Unlock() - return u.hits[path] -} - -const goPseudoVersion = "v0.0.0-20240102150405-abcdefabcdef" - -func goImmutableUpstream(t *testing.T) *countingUpstream { - return newCountingUpstream(t, func(path string) (string, bool) { - rest, ok := strings.CutPrefix(path, "/example.com/!mod/") - if !ok { - return "", false - } - switch { - case rest == "@v/list": - return "v1.0.0\n", true - case rest == "@latest", rest == "@v/main.info": - return `{"Version":"v1.0.0","Time":"2020-01-01T00:00:00Z"}`, true - case rest == "@v/v1.0.0.info": - return `{"Version":"v1.0.0","Time":"2020-01-01T00:00:00Z"}`, true - case rest == "@v/"+goPseudoVersion+".info": - return `{"Version":"` + goPseudoVersion + `","Time":"2024-01-02T15:04:05Z"}`, true - case strings.HasSuffix(rest, ".mod"): - return "module example.com/Mod\n", true - } - return "", false - }) -} - -// TestGoVersionFilesAreNotRevalidated checks that a canonical version's .mod -// and .info are fetched once and then served from the cache, with or without -// cache_metadata and whatever the metadata TTL, while queries, @v/list and -// @latest still go upstream. -func TestGoVersionFilesAreNotRevalidated(t *testing.T) { - for _, tt := range []struct { - name string - cacheMetadata, cooldown bool - }{ - {"cache_metadata off", false, false}, - {"cache_metadata on", true, false}, - {"cache_metadata off, cooldown", false, true}, - {"cache_metadata on, cooldown", true, true}, - } { - t.Run(tt.name, func(t *testing.T) { - testGoVersionFilesAreNotRevalidated(t, tt.cacheMetadata, tt.cooldown) - }) - } -} - -func testGoVersionFilesAreNotRevalidated(t *testing.T, cacheMetadata, withCooldown bool) { - u := goImmutableUpstream(t) - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = cacheMetadata - proxy.MetadataTTL = time.Nanosecond // every TTL-bound entry is already expired - if withCooldown { - proxy.Cooldown = &cooldown.Config{Default: "3d"} - } - h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes() - - once := []string{ - "/example.com/!mod/@v/v1.0.0.mod", - "/example.com/!mod/@v/v1.0.0.info", - "/example.com/!mod/@v/" + goPseudoVersion + ".mod", - "/example.com/!mod/@v/" + goPseudoVersion + ".info", - } - every := []string{ - "/example.com/!mod/@v/main.info", - "/example.com/!mod/@v/list", - "/example.com/!mod/@latest", - } - for range 3 { - for _, p := range append(append([]string{}, once...), every...) { - if w := goGet(h, p); w.Code != http.StatusOK { - t.Fatalf("GET %s = %d: %s", p, w.Code, w.Body.String()) - } - } - } - for _, p := range once { - if w := goGet(h, p); w.Header().Get("Warning") != "" { - t.Errorf("GET %s marked stale", p) - } - if got := u.count(p); got != 1 { - t.Errorf("upstream requests for %s = %d, want 1", p, got) - } - } - for _, p := range every { - if got := u.count(p); got < 2 { - t.Errorf("upstream requests for %s = %d, want one per request", p, got) - } - } -} - -func TestGoVersionImmutable(t *testing.T) { - tests := map[string]bool{ - "v1.0.0": true, - "v1.0.0-!r!c1": true, // v1.0.0-RC1 - "v2.0.0+incompatible": true, - goPseudoVersion: true, - "v1.2": false, // resolves to v1.2.0 - "v1.0.0+build": false, - "main": false, - "latest": false, - "v1.0.0-RC1": false, // not a valid escaped version - "": false, - "master-20240102150405-x": false, - } - for v, want := range tests { - if got := goVersionImmutable(v); got != want { - t.Errorf("goVersionImmutable(%q) = %v, want %v", v, got, want) - } - } -} - -// TestImmutableMetadataRefetchedWhenCorrupt checks that a stored immutable -// document whose bytes no longer match their digest is fetched again rather -// than returned for good. (A streamed hit of such a document aborts, as -// TestCachedMetadataHitAbortsOnDigestMismatch checks.) -func TestImmutableMetadataRefetchedWhenCorrupt(t *testing.T) { - u := goImmutableUpstream(t) - proxy, db, store, _ := setupTestProxy(t) - const ( - path = "/example.com/!mod/@v/v1.0.0.info" - key = "example.com/Mod/@v/v1.0.0.info" - want = `{"Version":"v1.0.0","Time":"2020-01-01T00:00:00Z"}` - ) - fetch := func() string { - t.Helper() - res := proxy.fetchImmutableMetadata(context.Background(), "golang", key, u.server.URL+path, "*/*") - if res.err != nil { - t.Fatalf("fetch: %v", res.err) - } - return string(res.body) - } - - fetch() - entry, err := db.GetMetadataCache("golang", key) - if err != nil || entry == nil { - t.Fatalf("document not stored: %v", err) - } - if _, _, err := store.Store(context.Background(), entry.StoragePath, strings.NewReader(`{"Version":"v6.6.6"}`)); err != nil { - t.Fatal(err) - } - - if got := fetch(); got != want { - t.Fatalf("got %q, want the upstream document", got) - } - if got := u.count(path); got != 2 { - t.Errorf("upstream requests = %d, want 2", got) - } - if got := fetch(); got != want { - t.Fatalf("got %q after refetch, want the upstream document", got) - } - if got := u.count(path); got != 2 { - t.Errorf("upstream requests after refetch = %d, want 2", got) - } -} - -// TestRPMContentAddressedRepodataIsNotRevalidated checks that repodata files -// named after their checksum are fetched once, while repomd.xml and plainly -// named files follow the TTL. -func TestRPMContentAddressedRepodataIsNotRevalidated(t *testing.T) { - const ( - sum = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" - primary = "/releases/40/Everything/x86_64/os/repodata/" + sum + "-primary.xml.zst" - repomd = "/releases/40/Everything/x86_64/os/repodata/repomd.xml" - plain = "/releases/40/Everything/x86_64/os/repodata/primary.xml.gz" - ) - for _, cacheMetadata := range []bool{false, true} { - u := newCountingUpstream(t, func(string) (string, bool) { return "data", true }) - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = cacheMetadata - proxy.MetadataTTL = time.Nanosecond - h := NewRPMHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes() - - for range 3 { - for _, p := range []string{primary, repomd, plain} { - if w := goGet(h, p); w.Code != http.StatusOK || w.Body.String() != "data" { - t.Fatalf("GET %s = %d %q", p, w.Code, w.Body.String()) - } - } - } - if got := u.count(primary); got != 1 { - t.Errorf("cache_metadata=%v: upstream requests for checksum-named file = %d, want 1", cacheMetadata, got) - } - for _, p := range []string{repomd, plain} { - if got := u.count(p); got != 3 { - t.Errorf("cache_metadata=%v: upstream requests for %s = %d, want 3", cacheMetadata, p, got) - } - } - } -} - -func TestRPMContentAddressed(t *testing.T) { - sha256 := strings.Repeat("a", 64) - tests := map[string]bool{ - "os/repodata/" + sha256 + "-primary.xml.gz": true, - "os/repodata/" + strings.Repeat("0", 40) + "-other.xml.gz": true, - "os/repodata/" + strings.Repeat("f", 128) + "-comps.xml": true, - "os/repodata/repomd.xml": false, - "os/repodata/repomd.xml.asc": false, - "os/repodata/primary.xml.gz": false, - "os/repodata/" + strings.Repeat("a", 63) + "-primary.xml.gz": false, - "os/repodata/" + strings.Repeat("A", 64) + "-primary.xml.gz": false, - "os/repodata/" + sha256: false, - } - for p, want := range tests { - if got := rpmContentAddressed(p); got != want { - t.Errorf("rpmContentAddressed(%q) = %v, want %v", p, got, want) - } - } -} - -// TestImmutableMetadataOverSizeLimitIsStreamed checks that a checksum-named -// repodata file too large to buffer is streamed from upstream rather than -// refused. -func TestImmutableMetadataOverSizeLimitIsStreamed(t *testing.T) { - const path = "/os/repodata/" + "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" + "-filelists.xml.gz" - u := newCountingUpstream(t, func(string) (string, bool) { return "large document", true }) - proxy, db, _, _ := setupTestProxy(t) - proxy.MetadataMaxSize = 4 - h := NewRPMHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes() - - if w := goGet(h, path); w.Code != http.StatusOK || w.Body.String() != "large document" { - t.Fatalf("got %d %q, want the upstream document", w.Code, w.Body.String()) - } - if entry, _ := db.GetMetadataCache("rpm", hashedCacheKey(u.server.URL, strings.TrimPrefix(path, "/"))); entry != nil { - t.Error("stored a document over the size limit") - } -} diff --git a/internal/handler/metadata_not_modified_test.go b/internal/handler/metadata_not_modified_test.go deleted file mode 100644 index fd690df..0000000 --- a/internal/handler/metadata_not_modified_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package handler - -import ( - "context" - "database/sql" - "net/http" - "net/http/httptest" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/database" -) - -// TestMetadataNotModifiedKeepsStoredDocument checks that a 304 from upstream -// refreshes the cache row without writing the unchanged document to storage -// again, and still serves it. -func TestMetadataNotModifiedKeepsStoredDocument(t *testing.T) { - const body = `{"v":1}` - var notModified atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(headerETag, `"v1"`) - if r.Header.Get("If-None-Match") == `"v1"` { - notModified.Add(1) - w.WriteHeader(http.StatusNotModified) - return - } - w.Header().Set(headerContentType, contentTypeJSON) - _, _ = w.Write([]byte(body)) - })) - t.Cleanup(upstream.Close) - - proxy, db, store, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Nanosecond // revalidate on every request - proxy.HTTPClient = upstream.Client() - - ctx := context.Background() - first := proxy.fetchMetadataDocument(ctx, "test", "pkg", upstream.URL+"/pkg", contentTypeJSON, nil) - if first.err != nil { - t.Fatal(first.err) - } - stored, err := db.GetMetadataCache("test", "pkg") - if err != nil || stored == nil { - t.Fatalf("cache row after first fetch: %v, %v", stored, err) - } - - for range 2 { - res := proxy.fetchMetadataDocument(ctx, "test", "pkg", upstream.URL+"/pkg", contentTypeJSON, nil) - if res.err != nil || string(res.body) != body || res.digest != first.digest { - t.Fatalf("revalidated fetch = %q, digest %q, err %v; want %q, digest %q", res.body, res.digest, res.err, body, first.digest) - } - } - if n := notModified.Load(); n != 2 { - t.Fatalf("upstream answered 304 %d times, want 2", n) - } - if store.stores != 1 { - t.Errorf("document stored %d times, want once", store.stores) - } - refreshed, err := db.GetMetadataCache("test", "pkg") - if err != nil { - t.Fatal(err) - } - if !refreshed.FetchedAt.Time.After(stored.FetchedAt.Time) { - t.Errorf("fetched_at = %v, want later than %v", refreshed.FetchedAt.Time, stored.FetchedAt.Time) - } - if refreshed.ContentDigest != stored.ContentDigest || refreshed.ETag != stored.ETag { - t.Errorf("row validators changed: %+v, want %+v", refreshed, stored) - } -} - -// TestMetadataNotModifiedStoresRowWithoutDigest checks that a 304 for a row -// written before digests were recorded stores the document as before, so the -// row gains its digest. -func TestMetadataNotModifiedStoresRowWithoutDigest(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("If-None-Match") == `"v1"` { - w.WriteHeader(http.StatusNotModified) - return - } - t.Errorf("unconditional upstream request") - })) - t.Cleanup(upstream.Close) - - proxy, db, store, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Nanosecond - proxy.HTTPClient = upstream.Client() - - path := metadataStoragePath("test", "pkg") - store.files[path] = []byte(`{"v":1}`) - if err := db.UpsertMetadataCache(&database.MetadataCacheEntry{ - Ecosystem: "test", - Name: "pkg", - StoragePath: path, - ETag: sql.NullString{String: `"v1"`, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, - }); err != nil { - t.Fatal(err) - } - - res := proxy.fetchMetadataDocument(context.Background(), "test", "pkg", upstream.URL+"/pkg", contentTypeJSON, nil) - if res.err != nil || string(res.body) != `{"v":1}` { - t.Fatalf("fetch = %q, %v", res.body, res.err) - } - if want := "sha256:" + sha256Hex(`{"v":1}`); res.digest != want { - t.Errorf("digest = %q, want %q", res.digest, want) - } - if store.stores != 1 { - t.Errorf("document stored %d times, want once", store.stores) - } -} diff --git a/internal/handler/metadata_path.go b/internal/handler/metadata_path.go deleted file mode 100644 index 4637909..0000000 --- a/internal/handler/metadata_path.go +++ /dev/null @@ -1,134 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - - "github.com/git-pkgs/proxy/internal/database" -) - -// metadataStoragePath builds the storage path for cached metadata. Cache -// keys are case-sensitive, but on a case-insensitive filesystem (APFS, NTFS) -// keys that differ only in case, such as Go's github.com/Sirupsen/logrus and -// github.com/sirupsen/logrus, would share one file. Keys may also hold names -// Windows reserves or end in a dot. So the path carries a hash of the exact -// key. Keys that already are lowercase SHA-256 hex digests are used as they -// are, so their existing entries stay valid. -func metadataStoragePath(ecosystem, cacheKey string) string { - if !isSHA256Hex(cacheKey) { - digest := sha256.Sum256([]byte(cacheKey)) - cacheKey = hex.EncodeToString(digest[:]) - } - return legacyMetadataStoragePath(ecosystem, cacheKey) -} - -// legacyMetadataStoragePath is where metadata was stored before paths were -// hashed: under the cache key itself. -func legacyMetadataStoragePath(ecosystem, cacheKey string) string { - return "_metadata/" + ecosystem + "/" + cacheKey + "/metadata" -} - -func isSHA256Hex(s string) bool { - if len(s) != 2*sha256.Size { - return false - } - for i := range len(s) { - c := s[i] - if (c < '0' || c > '9') && (c < 'a' || c > 'f') { - return false - } - } - return true -} - -// migrateLegacyMetadataEntry handles a cache row whose blob is still at its -// legacy, unhashed path. On a case-insensitive filesystem keys differing only -// in case shared that blob, so it is served only if it matches the row's digest. -func (p *Proxy) migrateLegacyMetadataEntry(ctx context.Context, ecosystem, cacheKey string, entry *database.MetadataCacheEntry) *database.MetadataCacheEntry { - if entry == nil || p.Storage == nil { - return entry - } - legacy := legacyMetadataStoragePath(ecosystem, cacheKey) - hashed := metadataStoragePath(ecosystem, cacheKey) - if entry.StoragePath != legacy || legacy == hashed { - return entry - } - var matches bool - if entry.ContentDigest.Valid { - digest, err := p.storedDigest(ctx, legacy) - if err != nil { - p.Logger.Warn("failed to read legacy metadata blob", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return nil - } - matches = digest == entry.ContentDigest.String - } - if !matches { - if err := p.DB.ForgetMetadataDocument(ecosystem, cacheKey, legacy, hashed); err != nil { - p.Logger.Warn("failed to drop legacy metadata row", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return nil - } - p.queueLegacyMetadataBlob(ecosystem, cacheKey, legacy) - return nil - } - if !p.moveLegacyMetadataBlob(ctx, entry, legacy, hashed) { - return entry - } - p.queueLegacyMetadataBlob(ecosystem, cacheKey, legacy) - moved := *entry - moved.StoragePath = hashed - return &moved -} - -// storedDigest hashes the blob at path as cacheMetadataBlob records it. -func (p *Proxy) storedDigest(ctx context.Context, path string) (string, error) { - r, err := p.Storage.Open(ctx, path) - if err != nil { - return "", err - } - defer func() { _ = r.Close() }() - h := sha256.New() - if _, err := io.Copy(h, r); err != nil { - return "", err - } - return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil -} - -// moveLegacyMetadataBlob copies entry's verified blob from legacy to hashed -// and points the row there, reporting whether it did. -func (p *Proxy) moveLegacyMetadataBlob(ctx context.Context, entry *database.MetadataCacheEntry, legacy, hashed string) bool { - logFailure := func(err error) { - p.Logger.Warn("failed to move legacy metadata blob", "ecosystem", entry.Ecosystem, "key", entry.Name, "error", err) - } - r, err := p.Storage.Open(ctx, legacy) - if err != nil { - logFailure(err) - return false - } - defer func() { _ = r.Close() }() - _, hash, err := p.Storage.Store(ctx, hashed, r) - if err != nil { - logFailure(err) - return false - } - if "sha256:"+hash != entry.ContentDigest.String { - logFailure(fmt.Errorf("blob changed while being copied")) - return false - } - moved, err := p.DB.MoveMetadataCache(entry.Ecosystem, entry.Name, legacy, hashed, entry.ContentDigest.String) - if err != nil { - logFailure(err) - return false - } - return moved -} - -// queueLegacyMetadataBlob queues a row's legacy blob for deletion once the row -// no longer reads it. Reclaim skips it while another row still points there. -func (p *Proxy) queueLegacyMetadataBlob(ecosystem, cacheKey, legacy string) { - if err := p.DB.QueuePendingDelete(legacy); err != nil { - p.Logger.Warn("failed to queue legacy metadata blob for deletion", "ecosystem", ecosystem, "key", cacheKey, "error", err) - } -} diff --git a/internal/handler/metadata_path_test.go b/internal/handler/metadata_path_test.go deleted file mode 100644 index b1f040c..0000000 --- a/internal/handler/metadata_path_test.go +++ /dev/null @@ -1,351 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "crypto/sha256" - "database/sql" - "encoding/hex" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "path/filepath" - "slices" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/storage" - "github.com/git-pkgs/registries/fetch" -) - -func TestMetadataStoragePathKeepsCaseApart(t *testing.T) { - for _, pair := range [][2]string{ - {"github.com/Sirupsen/logrus", "github.com/sirupsen/logrus"}, - {"JSONStream", "jsonstream"}, - } { - a, b := metadataStoragePath("go", pair[0]), metadataStoragePath("go", pair[1]) - if strings.EqualFold(a, b) { - t.Errorf("%q and %q map to paths %q and %q, which a case-insensitive filesystem treats as one", pair[0], pair[1], a, b) - } - } - for _, key := range []string{"CON", "nul", "trailing.", "a/../b"} { - p := metadataStoragePath("npm", key) - if strings.Contains(p, key) { - t.Errorf("path %q contains the raw key %q", p, key) - } - } - // Keys that already are digests keep their path, so existing entries - // stay valid. - sum := sha256.Sum256([]byte("generic")) - digest := hex.EncodeToString(sum[:]) - if got, want := metadataStoragePath("generic", digest), "_metadata/generic/"+digest+"/metadata"; got != want { - t.Errorf("digest key path = %q, want %q", got, want) - } -} - -// fileStorageProxy returns a proxy whose metadata is cached on the local -// filesystem, as with file:// storage. -func fileStorageProxy(t *testing.T, client *http.Client) (*Proxy, *database.DB) { - t.Helper() - dir := t.TempDir() - db, err := database.Create(filepath.Join(dir, "test.db")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = db.Close() }) - store, err := storage.OpenBucket(context.Background(), "file://"+filepath.Join(dir, "cache")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = store.Close() }) - proxy := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil))) - proxy.HTTPClient = client - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - return proxy, db -} - -// TestMetadataKeysDifferingInCaseCachedApart caches metadata for two keys -// that differ only in case on file storage. On a case-insensitive filesystem -// they shared one file, and the first key was served the second's document. -func TestMetadataKeysDifferingInCaseCachedApart(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - _, _ = io.WriteString(w, "document for "+r.URL.Path) - })) - defer upstream.Close() - proxy, _ := fileStorageProxy(t, upstream.Client()) - - keys := []string{"github.com/Sirupsen/logrus", "github.com/sirupsen/logrus"} - for range 2 { - for _, key := range keys { - body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/"+key) - if err != nil { - t.Fatal(err) - } - if want := "document for /" + key; string(body) != want { - t.Errorf("%s: served %q, want %q", key, body, want) - } - } - } - if got := requests.Load(); got != 2 { - t.Errorf("upstream requests = %d, want 2", got) - } -} - -// seedLegacyMetadata stores blob at key's legacy path and a row pointing at -// it, fetched at fetchedAt. The row's digest is that of document, or unset -// when document is empty. -func seedLegacyMetadata(t *testing.T, proxy *Proxy, db *database.DB, key, blob, document string, fetchedAt time.Time) string { - t.Helper() - legacy := legacyMetadataStoragePath("go", key) - if _, _, err := proxy.Storage.Store(context.Background(), legacy, strings.NewReader(blob)); err != nil { - t.Fatal(err) - } - entry := &database.MetadataCacheEntry{ - Ecosystem: "go", - Name: key, - StoragePath: legacy, - ETag: sql.NullString{String: `"v1"`, Valid: true}, - Size: sql.NullInt64{Int64: int64(len(blob)), Valid: true}, - FetchedAt: sql.NullTime{Time: fetchedAt, Valid: true}, - } - if document != "" { - sum := sha256.Sum256([]byte(document)) - entry.ContentDigest = sql.NullString{String: "sha256:" + hex.EncodeToString(sum[:]), Valid: true} - } - if err := db.UpsertMetadataCache(entry); err != nil { - t.Fatal(err) - } - return legacy -} - -func pendingDeletes(t *testing.T, db *database.DB) []string { - t.Helper() - due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 100) - if err != nil { - t.Fatal(err) - } - return due -} - -func metadataRowPath(t *testing.T, db *database.DB, key string) string { - t.Helper() - entry, err := db.GetMetadataCache("go", key) - if err != nil || entry == nil { - t.Fatalf("cache row for %s: %v %v", key, entry, err) - } - return entry.StoragePath -} - -// TestLegacyMetadataMatchingBlobServedWhenUpstreamDown checks that a row at -// its legacy path whose blob matches its digest is still served as a stale -// fallback, and is moved to the hashed path. -func TestLegacyMetadataMatchingBlobServedWhenUpstreamDown(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusBadGateway) - })) - defer upstream.Close() - proxy, db := fileStorageProxy(t, upstream.Client()) - - const key = "github.com/Sirupsen/logrus" - legacy := seedLegacyMetadata(t, proxy, db, key, "old", "old", time.Now().Add(-2*time.Hour)) - - for range 2 { - body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x") - if err != nil { - t.Fatalf("legacy document not served while upstream is down: %v", err) - } - if string(body) != "old" { - t.Fatalf("served %q, want %q", body, "old") - } - } - if got, want := metadataRowPath(t, db, key), metadataStoragePath("go", key); got != want { - t.Errorf("storage path = %q, want %q", got, want) - } - if due := pendingDeletes(t, db); !slices.Contains(due, legacy) { - t.Errorf("pending deletes = %v, want %q", due, legacy) - } -} - -// TestLegacyMetadataMatchingBlobServedFromCache checks that a fresh row at -// its legacy path whose blob matches is served without asking upstream. -func TestLegacyMetadataMatchingBlobServedFromCache(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - _, _ = io.WriteString(w, "fresh") - })) - defer upstream.Close() - proxy, db := fileStorageProxy(t, upstream.Client()) - - const key = "github.com/Sirupsen/logrus" - seedLegacyMetadata(t, proxy, db, key, "old", "old", time.Now()) - - body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x") - if err != nil { - t.Fatal(err) - } - if string(body) != "old" || requests.Load() != 0 { - t.Errorf("served %q after %d upstream requests, want the cached document", body, requests.Load()) - } -} - -// TestLegacyMetadataMismatchedBlobNotServed checks that a legacy blob that -// does not match the row's digest, as when another key's document -// overwrote it, is neither served nor used as a fallback. -func TestLegacyMetadataMismatchedBlobNotServed(t *testing.T) { - var down atomic.Bool - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - if down.Load() { - w.WriteHeader(http.StatusBadGateway) - return - } - if r.Header.Get("If-None-Match") != "" { - t.Errorf("revalidated a document that was dropped") - } - _, _ = io.WriteString(w, "fresh") - })) - defer upstream.Close() - proxy, db := fileStorageProxy(t, upstream.Client()) - - const key = "github.com/Sirupsen/logrus" - seedLegacyMetadata(t, proxy, db, key, "other key's document", "old", time.Now()) - down.Store(true) - if body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x"); err == nil { - t.Fatalf("served %q, want an error while upstream is down", body) - } - - down.Store(false) - body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x") - if err != nil { - t.Fatal(err) - } - if string(body) != "fresh" { - t.Errorf("served %q, want %q", body, "fresh") - } - if got, want := metadataRowPath(t, db, key), metadataStoragePath("go", key); got != want { - t.Errorf("storage path = %q, want %q", got, want) - } -} - -// caseFoldingStorage stores paths differing only in case as one file, as a -// case-insensitive filesystem does. -type caseFoldingStorage struct{ storage.Storage } - -func (s caseFoldingStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { - return s.Storage.Store(ctx, strings.ToLower(path), r) -} - -func (s caseFoldingStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) { - return s.Storage.Open(ctx, strings.ToLower(path)) -} - -func (s caseFoldingStorage) Exists(ctx context.Context, path string) (bool, error) { - return s.Storage.Exists(ctx, strings.ToLower(path)) -} - -func (s caseFoldingStorage) Delete(ctx context.Context, path string) error { - return s.Storage.Delete(ctx, strings.ToLower(path)) -} - -func (s caseFoldingStorage) Size(ctx context.Context, path string) (int64, error) { - return s.Storage.Size(ctx, strings.ToLower(path)) -} - -// TestLegacyMetadataCaseVariantRefetchedAfterReclaim checks that on -// case-insensitive storage, a key whose legacy blob was shared with a case -// variant and deleted after that variant moved is fetched again rather than -// served. -func TestLegacyMetadataCaseVariantRefetchedAfterReclaim(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - _, _ = io.WriteString(w, "fresh") - })) - defer upstream.Close() - proxy, db := fileStorageProxy(t, upstream.Client()) - proxy.Storage = caseFoldingStorage{proxy.Storage} - - const upper, lower = "github.com/Sirupsen/logrus", "github.com/sirupsen/logrus" - seedLegacyMetadata(t, proxy, db, upper, "doc", "doc", time.Now()) - seedLegacyMetadata(t, proxy, db, lower, "doc", "doc", time.Now()) - - if body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", upper, upstream.URL+"/x"); err != nil || string(body) != "doc" { - t.Fatalf("FetchOrCacheMetadata(%s) = %q, %v; want the migrated document", upper, body, err) - } - // Reclaim deletes the queued legacy blob, which is also lower's. - for _, path := range pendingDeletes(t, db) { - if err := proxy.Storage.Delete(context.Background(), path); err != nil { - t.Fatal(err) - } - } - - body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", lower, upstream.URL+"/x") - if err != nil { - t.Fatal(err) - } - if string(body) != "fresh" || requests.Load() != 1 { - t.Errorf("served %q after %d upstream requests, want a fresh fetch", body, requests.Load()) - } - if got, want := metadataRowPath(t, db, lower), metadataStoragePath("go", lower); got != want { - t.Errorf("storage path = %q, want %q", got, want) - } -} - -// TestLegacyMetadataPathRefetched checks that a row at its legacy path with -// no digest to check its blob against is fetched again, moved to the hashed -// path, and its old blob queued for deletion. -func TestLegacyMetadataPathRefetched(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - _, _ = io.WriteString(w, "fresh") - })) - defer upstream.Close() - proxy, db := fileStorageProxy(t, upstream.Client()) - - const key = "github.com/Sirupsen/logrus" - legacy := legacyMetadataStoragePath("go", key) - if _, _, err := proxy.Storage.Store(context.Background(), legacy, bytes.NewReader([]byte("old"))); err != nil { - t.Fatal(err) - } - if err := db.UpsertMetadataCache(&database.MetadataCacheEntry{ - Ecosystem: "go", - Name: key, - StoragePath: legacy, - Size: sql.NullInt64{Int64: 3, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - }); err != nil { - t.Fatal(err) - } - - body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x") - if err != nil { - t.Fatal(err) - } - if string(body) != "fresh" || requests.Load() != 1 { - t.Fatalf("served %q after %d upstream requests, want a fresh fetch", body, requests.Load()) - } - entry, err := db.GetMetadataCache("go", key) - if err != nil || entry == nil { - t.Fatalf("cache row: %v %v", entry, err) - } - if entry.StoragePath != metadataStoragePath("go", key) { - t.Errorf("storage path = %q, want %q", entry.StoragePath, metadataStoragePath("go", key)) - } - due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 10) - if err != nil { - t.Fatal(err) - } - if !slices.Contains(due, legacy) { - t.Errorf("pending deletes = %v, want %q", due, legacy) - } -} diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go deleted file mode 100644 index fc16d43..0000000 --- a/internal/handler/metadata_stored.go +++ /dev/null @@ -1,313 +0,0 @@ -package handler - -import ( - "context" - "crypto/rand" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "hash" - "io" - "net/http" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/metrics" -) - -// A metadata document can be rewritten from storage instead of from memory: -// storedMetadataDocument streams it from upstream into storage, and the -// rewrite reads it back as often as it needs. Nothing holds the whole -// document, so its size is not limited by metadata_max_size or by memory. - -// errMetadataNotStored is returned when metadata is not cached in storage, so -// a document has to be read into memory. -var errMetadataNotStored = errors.New("metadata is not cached in storage") - -// errStoredMetadataChanged is returned when a stored document no longer has -// the digest its cache row records: a fetch replaced it while it was read. -var errStoredMetadataChanged = errors.New("stored metadata changed while it was read") - -// storedMetadataDocument returns the cache row of a metadata document stored -// uncompressed with its digest: the stored copy while it is within its TTL, -// or else the document fetched from upstream and stored, or the stored copy -// when upstream fails. The document is streamed from upstream into storage -// without being held in memory, so a fetch takes no metadata_max_size limit. -// Concurrent calls for one document share a fetch, and instances sharing the -// database take turns as for other metadata. -// -// validate, when not nil, checks a fetched document before it replaces the -// stored one; a document it rejects is not stored, and the stored copy is -// served as when upstream fails. It does not check a stored copy within its -// TTL. -// -// It returns errMetadataNotStored when the proxy doesn't cache metadata or -// storing the document failed, and the caller reads it into memory instead. -func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error) (*database.MetadataCacheEntry, error) { - res := p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, "") - return res.entry, res.err -} - -// storedMetadata implements storedMetadataDocument, and reports in the -// result whether the row was read from cache or served stale. A row with -// the digest gone, one whose stored copy was missing or out of date when -// read, is not served or revalidated, within its TTL or not, and the -// document is fetched again. -func (p *Proxy) storedMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone string) metadataResult { - if !p.CacheMetadata || p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 { - return metadataResult{err: errMetadataNotStored} - } - if containsPathTraversal(cacheKey) { - return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} - } - if entry, fresh := p.storedMetadataEntryState(ctx, ecosystem, cacheKey, gone); fresh { - metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) - p.recordMetadataRead(entry) - return metadataResult{entry: entry, fromCache: true} - } - p.recordMetadataCacheMiss(ecosystem) - - key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, "", false) + "\x00stored" - res, _ := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { - entry, fresh := p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey, gone) - if fresh { - return metadataResult{entry: entry} - } - release, outcome, err := p.awaitFetchLease(fetchCtx, metrics.CacheKindMetadata, key, func() bool { - entry, fresh = p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey, gone) - return fresh - }) - if err != nil { - return metadataResult{err: err} - } - switch outcome { - case leaseStored: - return metadataResult{entry: entry} - case leaseFailed: - // Serve the stored copy, as the holder did, rather than trying - // the failing upstream again. - p.reloadUpstreamHolds() - if usableStoredMetadata(entry) { - return p.staleStoredMetadata(ecosystem, cacheKey, entry, errFetchFailedElsewhere) - } - case leaseFetch: - } - res := metadataResult{err: errSharedFetchAbandoned} - defer func() { release(res.err != nil || res.stale) }() - res = p.fetchStoredMetadata(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, validate, entry) - return res - }) - return res -} - -// storedMetadataEntryState reads the cache row of a document and reports -// whether it is within its TTL and usable by storedMetadataDocument. A row -// with the digest gone is returned as nil. -func (p *Proxy) storedMetadataEntryState(ctx context.Context, ecosystem, cacheKey, gone string) (*database.MetadataCacheEntry, bool) { - entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) - entry = p.migrateLegacyMetadataEntry(ctx, ecosystem, cacheKey, entry) - if gone != "" && entry != nil && entry.ContentDigest.String == gone { - return nil, false - } - fresh := usableStoredMetadata(entry) && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL - return entry, fresh -} - -// usableStoredMetadata reports whether entry describes a document stored -// uncompressed with its digest, which a rewrite can read back and check. -func usableStoredMetadata(entry *database.MetadataCacheEntry) bool { - return entry != nil && entry.ContentDigest.Valid && entry.ContentEncoding.String == "" -} - -// fetchStoredMetadata fetches a document from upstream into storage, -// revalidating entry when there is one, and falls back to entry when -// upstream fails or validate rejects what it sent. A failure to store is -// returned as it is, for the caller to read the document into memory. -func (p *Proxy) fetchStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, entry *database.MetadataCacheEntry) metadataResult { - revalidate := entry - if !usableStoredMetadata(entry) { - revalidate = nil - } - stored, err := p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, revalidate) - if errors.Is(err, errStale304) { - stored, err = p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, nil) - } - if err == nil { - return metadataResult{entry: stored} - } - // As in fetchMetadataFromUpstream, any upstream failure, a 404 included, - // falls back to the stored copy. - if !usableStoredMetadata(entry) || errors.Is(err, errMetadataNotStored) { - return metadataResult{err: err} - } - return p.staleStoredMetadata(ecosystem, cacheKey, entry, err) -} - -// staleStoredMetadata falls back to the stored document entry describes after -// the upstream fetch failed with err. -func (p *Proxy) staleStoredMetadata(ecosystem, cacheKey string, entry *database.MetadataCacheEntry, err error) metadataResult { - p.Logger.Warn("upstream metadata fetch failed, serving cached copy", - "ecosystem", ecosystem, "key", cacheKey, "error", err) - entry = p.currentMetadataEntry(ecosystem, cacheKey, entry) - if !usableStoredMetadata(entry) { - return metadataResult{err: fmt.Errorf("upstream failed and no usable cached metadata: %w", err)} - } - return metadataResult{entry: entry, stale: true, upstreamErr: err} -} - -// storeUpstreamMetadata fetches a document and streams it into storage, and -// returns its new cache row. On a 304 for entry it only moves the row's -// fetched_at on; errStale304 means the stored copy is gone or the row was -// replaced meanwhile, and the caller should fetch without validators. -// -// With validate, the document is staged under a path of its own and checked -// there first, so one validate rejects never replaces the stored copy. -func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, entry *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { - resp, err := p.requestUpstreamMetadata(ctx, upstreamURL, entry, accept, "") - if err != nil { - return nil, err - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode == http.StatusNotModified && entry != nil { - if ok, err := p.Storage.Exists(ctx, entry.StoragePath); err != nil || !ok { - return nil, errStale304 - } - refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) - if err != nil || !refreshed { - return nil, errStale304 - } - return p.currentMetadataEntry(ecosystem, cacheKey, entry), nil - } - if err := upstreamMetadataStatus(resp); err != nil { - return nil, err - } - - meta := &upstreamMetadata{ - contentType: resp.Header.Get(headerContentType), - etag: resp.Header.Get(headerETag), - } - if meta.contentType == "" { - meta.contentType = contentTypeJSON - } - if lm := resp.Header.Get(headerLastModified); lm != "" { - meta.lastModified, _ = http.ParseTime(lm) - } - limit := p.MetadataStreamMaxSize - if limit <= 0 { - limit = defaultMetadataStreamMaxSize - } - // Go decompresses the body, since no Accept-Encoding was set, so it is - // stored uncompressed. - var body io.Reader = &limitedBody{r: upstreamBodyReader{resp.Body}, left: limit} - if validate != nil { - staged, err := p.stageMetadata(ctx, ecosystem, cacheKey, body, validate) - if err != nil { - return nil, err - } - defer func() { _ = staged.Close() }() - body = staged - } - if _, err := p.storeMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), body, meta); err != nil { - return nil, notStored("", err) - } - stored, err := p.DB.GetMetadataCache(ecosystem, cacheKey) - if err != nil || !usableStoredMetadata(stored) { - return nil, fmt.Errorf("reading the stored metadata row: %w", errors.Join(err, errMetadataNotStored)) - } - return stored, nil -} - -// stageMetadata stores body under a staging path of its own, checks it with -// validate, and returns it opened for reading, to be stored in place of the -// cached copy. The staged copy is deleted when the reader is closed. A -// failure to store it is returned as errMetadataNotStored. -func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, body io.Reader, validate func(io.Reader) error) (io.ReadCloser, error) { - path := metadataStoragePath(ecosystem, cacheKey) + ".staged-" + rand.Text() - discard := func() { - if err := p.Storage.Delete(context.WithoutCancel(ctx), path); err != nil { - p.Logger.Warn("failed to delete staged metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) - } - } - if _, _, err := p.Storage.Store(ctx, path, body); err != nil { - discard() - return nil, notStored("staging: ", err) - } - if err := readStored(func() (io.ReadCloser, error) { return p.Storage.Open(ctx, path) }, validate); err != nil { - discard() - return nil, err - } - staged, err := p.Storage.Open(ctx, path) - if err != nil { - discard() - return nil, fmt.Errorf("%w: reading staged copy: %w", errMetadataNotStored, err) - } - return &stagedReader{ReadCloser: staged, discard: discard}, nil -} - -// limitedBody fails with ErrMetadataTooLarge, as an upstream failure, once -// r has sent more than left bytes. -type limitedBody struct { - r io.Reader - left int64 -} - -func (l *limitedBody) Read(b []byte) (int, error) { - n, err := l.r.Read(b) - if l.left -= int64(n); l.left < 0 { - return 0, &upstreamBodyError{ErrMetadataTooLarge} - } - return n, err -} - -// notStored returns a failure to store a document as errMetadataNotStored, -// unless reading it from upstream failed. -func notStored(what string, err error) error { - if body := (*upstreamBodyError)(nil); errors.As(err, &body) { - return body - } - return fmt.Errorf("%w: %s%w", errMetadataNotStored, what, err) -} - -// stagedReader deletes the staged copy it reads when closed. -type stagedReader struct { - io.ReadCloser - discard func() -} - -func (s *stagedReader) Close() error { - err := s.ReadCloser.Close() - s.discard() - return err -} - -// openStoredMetadata opens the document entry describes. Reading it to the -// end fails with errStoredMetadataChanged when its bytes don't have the -// digest entry records. -func (p *Proxy) openStoredMetadata(ctx context.Context, entry *database.MetadataCacheEntry) (io.ReadCloser, error) { - r, err := p.Storage.Open(ctx, entry.StoragePath) - if err != nil { - return nil, err - } - return &digestReader{r: r, h: sha256.New(), want: entry.ContentDigest.String}, nil -} - -// digestReader checks at the end of its reader that what it read has the -// digest want, in the form the metadata cache records it. -type digestReader struct { - r io.ReadCloser - h hash.Hash - want string -} - -func (d *digestReader) Read(b []byte) (int, error) { - n, err := d.r.Read(b) - d.h.Write(b[:n]) - if errors.Is(err, io.EOF) && "sha256:"+hex.EncodeToString(d.h.Sum(nil)) != d.want { - return n, errStoredMetadataChanged - } - return n, err -} - -func (d *digestReader) Close() error { return d.r.Close() } diff --git a/internal/handler/metadata_stream.go b/internal/handler/metadata_stream.go deleted file mode 100644 index 56606e2..0000000 --- a/internal/handler/metadata_stream.go +++ /dev/null @@ -1,134 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "net/http" - "strconv" - "strings" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/metrics" -) - -// serveCachedMetadata answers a ProxyCached request from a fresh metadata -// cache row by streaming the stored document, rather than reading it into -// memory first, so concurrent hits on a large document don't each hold a copy -// of it. The bytes are checked against the row's digest as they are sent, and -// a mismatch aborts the response and drops the row, the way a corrupt cached -// artifact is handled. -// -// It reports false, having written nothing, when the request should take the -// buffered path instead: no fresh row, a row without a digest or size, a -// document over the metadata size limit, or one that can't be opened. -func (p *Proxy) serveCachedMetadata(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string) bool { - return p.serveStoredMetadata(w, r, ecosystem, cacheKey, false) -} - -// serveStoredMetadata is serveCachedMetadata. With immutable set the row is -// served whatever its age and never marked stale, for documents that can't -// change upstream (see proxyImmutable). -func (p *Proxy) serveStoredMetadata(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, immutable bool) bool { - entry := p.storedMetadataEntry(ecosystem, cacheKey, immutable) - if entry == nil { - return false - } - body, err := p.openMetadataEntry(r.Context(), ecosystem, cacheKey, entry) - if err != nil { - return false - } - defer func() { _ = body.Close() }() - - p.recordStreamedMetadataHit(ecosystem, entry) - - cm := p.cachedMetaFor(entry) - if immutable { - cm.stale = false - } - if writeMetadataValidators(w, r, cm) { - return true - } - contentType := contentTypeJSON - if entry.ContentType.Valid { - contentType = entry.ContentType.String - } - w.Header().Set(headerContentType, contentType) - w.Header().Set(headerContentLength, strconv.FormatInt(entry.Size.Int64, 10)) - if entry.ContentEncoding.String != "" { - w.Header().Set(headerContentEncoding, entry.ContentEncoding.String) - } - markStale(w, cm.stale) - w.WriteHeader(http.StatusOK) - if r.Method != http.MethodHead { - copyArtifactBody(w, body, entry.Size.Int64, false) - } - return true -} - -// freshMetadataEntry returns the metadata cache row for ecosystem and -// cacheKey when its document can be streamed: the row is within its TTL and -// records a sha256 digest and a size within the metadata size limit. It -// returns nil otherwise. -func (p *Proxy) freshMetadataEntry(ecosystem, cacheKey string) *database.MetadataCacheEntry { - return p.storedMetadataEntry(ecosystem, cacheKey, false) -} - -// storedMetadataEntry is freshMetadataEntry, ignoring the row's age when -// immutable is set. -func (p *Proxy) storedMetadataEntry(ecosystem, cacheKey string, immutable bool) *database.MetadataCacheEntry { - if p.DB == nil || p.Storage == nil || (!immutable && p.MetadataTTL <= 0) || containsPathTraversal(cacheKey) { - return nil - } - entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) - if err != nil || entry == nil || !entry.FetchedAt.Valid { - return nil - } - if !immutable && time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { - return nil - } - limit := p.MetadataMaxSize - if limit <= 0 { - limit = defaultMetadataMaxSize - } - if !entry.ContentDigest.Valid || !strings.HasPrefix(entry.ContentDigest.String, "sha256:") || !entry.Size.Valid || entry.Size.Int64 > limit { - return nil - } - return entry -} - -// openMetadataEntry opens the stored document of a row freshMetadataEntry -// returned. Its bytes are checked against the row's digest as they are read: -// a mismatch fails the read at the end and drops the row, so the next -// request fetches the document again. -func (p *Proxy) openMetadataEntry(ctx context.Context, ecosystem, cacheKey string, entry *database.MetadataCacheEntry) (io.ReadCloser, error) { - checks, err := newIntegrityChecks(strings.TrimPrefix(entry.ContentDigest.String, "sha256:"), "") - if err != nil { - return nil, err - } - stored, err := p.Storage.Open(ctx, entry.StoragePath) - if err != nil { - return nil, err - } - body, err := checks.wrapFailOnMismatch(stored, func(reason string) { - p.Logger.Error("cached metadata failed integrity check", - "ecosystem", ecosystem, "key", cacheKey, "path", entry.StoragePath, "reason", reason) - metrics.RecordIntegrityFailure(ecosystem, metrics.FailureDetail{ - Path: entry.StoragePath, Err: errors.New(reason)}) - if err := p.DB.DiscardMetadataCache(ecosystem, cacheKey, entry.StoragePath, entry.ContentDigest.String); err != nil { - p.Logger.Warn("failed to clear corrupt metadata from cache", "error", err) - } - }) - if err != nil { - _ = stored.Close() - return nil, err - } - return body, nil -} - -// recordStreamedMetadataHit counts a hit on a row served by streaming. -func (p *Proxy) recordStreamedMetadataHit(ecosystem string, entry *database.MetadataCacheEntry) { - p.recordMetadataRead(entry) - metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) -} diff --git a/internal/handler/metadata_stream_test.go b/internal/handler/metadata_stream_test.go deleted file mode 100644 index 702eea8..0000000 --- a/internal/handler/metadata_stream_test.go +++ /dev/null @@ -1,223 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "io" - "net/http" - "net/http/httptest" - "slices" - "strconv" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/storage" -) - -// stallingStorage hands out cached metadata readers that return the first -// head bytes and then block until release closes, so a test can see whether a -// response starts before the whole document has been read. -type stallingStorage struct { - storage.Storage - head int - release chan struct{} -} - -func (s *stallingStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) { - rc, err := s.Storage.Open(ctx, path) - if err != nil || !strings.HasPrefix(path, "_metadata/") { - return rc, err - } - return &stallingReader{ReadCloser: rc, left: s.head, release: s.release}, nil -} - -type stallingReader struct { - io.ReadCloser - left int - release chan struct{} -} - -func (r *stallingReader) Read(p []byte) (int, error) { - if r.left == 0 { - <-r.release - return r.ReadCloser.Read(p) - } - if len(p) > r.left { - p = p[:r.left] - } - n, err := r.ReadCloser.Read(p) - r.left -= n - return n, err -} - -// cachedMetadataServer serves doc through ProxyCached and caches it with one -// request. upstreamHits counts the requests that reached the upstream. -func cachedMetadataServer(t *testing.T, doc []byte) (proxy *Proxy, store *mockStorage, srv *httptest.Server, upstreamHits *atomic.Int32) { - t.Helper() - upstreamHits = &atomic.Int32{} - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - upstreamHits.Add(1) - w.Header().Set(headerContentType, "application/x-test") - w.Header().Set(headerETag, `"v1"`) - _, _ = w.Write(doc) - })) - t.Cleanup(upstream.Close) - - proxy, _, store, _ = setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - proxy.ProxyCached(w, r, upstream.URL+"/doc", "test", "doc", "*/*") - })) - t.Cleanup(srv.Close) - - resp, err := http.Get(srv.URL) - if err != nil { - t.Fatalf("populating cache: %v", err) - } - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("populating cache: status %d", resp.StatusCode) - } - return proxy, store, srv, upstreamHits -} - -func TestConcurrentMetadataCacheHitsStream(t *testing.T) { - const callers = 10 - doc := bytes.Repeat([]byte("0123456789abcdef"), 1<<18) // 4 MB - proxy, store, srv, _ := cachedMetadataServer(t, doc) - - gate := &stallingStorage{Storage: store, head: 64 << 10, release: make(chan struct{})} - proxy.Storage = gate - var releaseOnce sync.Once - release := func() { releaseOnce.Do(func() { close(gate.release) }) } - t.Cleanup(release) - - // Every hit has to start its response while its reader is stalled after - // the first 64 KB. A hit that reads the document into memory first can't - // send anything until release. - type result struct { - resp *http.Response - first []byte - err error - } - started := make(chan result, callers) - for range callers { - go func() { - resp, err := http.Get(srv.URL) - if err != nil { - started <- result{err: err} - return - } - first := make([]byte, 1024) - _, err = io.ReadFull(resp.Body, first) - started <- result{resp: resp, first: first, err: err} - }() - } - results := make([]result, 0, callers) - timeout := time.After(5 * time.Second) - streamed := 0 - for len(results) < callers { - select { - case res := <-started: - results = append(results, res) - case <-timeout: - // Let the stalled hits finish, so the server can shut down. - release() - timeout = nil - streamed = len(results) - } - } - release() - t.Cleanup(func() { - for _, res := range results { - if res.resp != nil { - _ = res.resp.Body.Close() - } - } - }) - if timeout == nil { - t.Fatalf("%d of %d concurrent cache hits started their response before the document was read to its end", streamed, callers) - } - - for _, res := range results { - if res.err != nil { - t.Fatalf("cache hit: %v", res.err) - } - rest, err := io.ReadAll(res.resp.Body) - _ = res.resp.Body.Close() - if err != nil { - t.Fatalf("reading cache hit: %v", err) - } - if body := slices.Concat(res.first, rest); !bytes.Equal(body, doc) { - t.Fatalf("cache hit returned %d bytes, want the %d-byte document", len(body), len(doc)) - } - if got := res.resp.Header.Get(headerContentType); got != "application/x-test" { - t.Errorf("Content-Type = %q, want application/x-test", got) - } - if got := res.resp.Header.Get(headerContentLength); got != strconv.Itoa(len(doc)) { - t.Errorf("Content-Length = %q, want %d", got, len(doc)) - } - if got := res.resp.Header.Get(headerETag); got != `"v1"` { - t.Errorf("ETag = %q, want %q", got, `"v1"`) - } - } - - req, _ := http.NewRequest(http.MethodGet, srv.URL, nil) - req.Header.Set("If-None-Match", `"v1"`) - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("conditional cache hit: %v", err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusNotModified { - t.Errorf("conditional cache hit: status %d, want 304", resp.StatusCode) - } -} - -func TestCachedMetadataHitAbortsOnDigestMismatch(t *testing.T) { - doc := bytes.Repeat([]byte("metadata "), 1<<14) - _, store, srv, upstreamHits := cachedMetadataServer(t, doc) - - corrupt := bytes.Clone(doc) - corrupt[len(corrupt)-1] ^= 0xff - store.mu.Lock() - store.files[metadataStoragePath("test", "doc")] = corrupt - store.mu.Unlock() - - resp, err := http.Get(srv.URL) - if err != nil { - t.Fatalf("cache hit: %v", err) - } - defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(resp.Body) - if err == nil { - t.Fatalf("cache hit completed with %d bytes of a document that doesn't match its digest, want the response aborted", len(body)) - } - if len(body) >= len(doc) { - t.Errorf("aborted response still delivered %d of %d bytes", len(body), len(doc)) - } - - // The corrupt row is dropped, so the next request fetches the document - // from upstream again instead of aborting until the TTL runs out. - resp, err = http.Get(srv.URL) - if err != nil { - t.Fatalf("request after abort: %v", err) - } - defer func() { _ = resp.Body.Close() }() - body, err = io.ReadAll(resp.Body) - if err != nil { - t.Fatalf("request after abort: %v", err) - } - if !bytes.Equal(body, doc) { - t.Errorf("request after abort got %d bytes, want the upstream document (%d bytes)", len(body), len(doc)) - } - if got := upstreamHits.Load(); got != 2 { - t.Errorf("upstream requests = %d, want 2 (populate, then refetch after the corrupt hit)", got) - } -} diff --git a/internal/handler/metadata_truncated_test.go b/internal/handler/metadata_truncated_test.go deleted file mode 100644 index 5044374..0000000 --- a/internal/handler/metadata_truncated_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package handler - -import ( - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - - "github.com/git-pkgs/cooldown" -) - -// cutMetadataUpstream serves one npm packument and, when cut is set, closes -// the connection halfway through the document it announced. -type cutMetadataUpstream struct { - *httptest.Server - mu sync.Mutex - body string - cut bool -} - -func newCutMetadataUpstream(t *testing.T, body string) *cutMetadataUpstream { - t.Helper() - u := &cutMetadataUpstream{body: body} - u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - u.mu.Lock() - body, cut := u.body, u.cut - u.mu.Unlock() - if r.URL.Path != "/pkg" { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Content-Length", fmt.Sprint(len(body))) - if cut { - _, _ = io.WriteString(w, body[:len(body)/2]) - return - } - _, _ = io.WriteString(w, body) - })) - t.Cleanup(u.Close) - return u -} - -func (u *cutMetadataUpstream) set(body string, cut bool) { - u.mu.Lock() - defer u.mu.Unlock() - u.body, u.cut = body, cut -} - -// truncPackument is an npm packument with n versions published long ago. -func truncPackument(n int) string { - var versions, times []string - for i := range n { - v := fmt.Sprintf("1.0.%d", i) - versions = append(versions, fmt.Sprintf(`%q:{"name":"pkg","version":%q,"dist":{"tarball":"https://registry.npmjs.org/pkg/-/pkg-%s.tgz"}}`, v, v, v)) - times = append(times, fmt.Sprintf(`%q:"2020-01-01T00:00:00Z"`, v)) - } - return `{"name":"pkg","dist-tags":{"latest":"1.0.0"},"versions":{` + strings.Join(versions, ",") + - `},"time":{` + strings.Join(times, ",") + `}}` -} - -// TestMetadataTruncatedUpstream checks that a document upstream cuts off -// before its Content-Length is neither served nor cached: the cached copy is -// served instead, with and without cooldown. -func TestMetadataTruncatedUpstream(t *testing.T) { - for _, withCooldown := range []bool{false, true} { - t.Run(map[bool]string{false: "no cooldown", true: "cooldown"}[withCooldown], func(t *testing.T) { - u := newCutMetadataUpstream(t, truncPackument(50)) - proxy, db, _, _ := setupTestProxy(t) - proxy.HTTPClient = u.Client() - proxy.CacheMetadata = true - if withCooldown { - proxy.Cooldown = &cooldown.Config{Default: "3d"} - } - handler := NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() - - first := serve(handler, "/pkg", nil) - if first.Code != http.StatusOK { - t.Fatalf("first request: status %d", first.Code) - } - before, err := db.GetMetadataCache("npm", "pkg") - if err != nil || before == nil { - t.Fatalf("no metadata row after the first request: %v", err) - } - - u.set(truncPackument(51), true) - expireMetadata(t, db, "npm", "pkg") - proxy.SetMetadataRewriteCacheSize(0) - - rec := serve(handler, "/pkg", nil) - if rec.Code != http.StatusOK || rec.Body.String() != first.Body.String() { - t.Errorf("status %d, want 200 with the cached document", rec.Code) - } - after, err := db.GetMetadataCache("npm", "pkg") - if err != nil || after == nil || after.ContentDigest != before.ContentDigest { - t.Error("the cut-off document replaced the cached copy") - } - }) - } -} - -// TestMetadataTruncatedUpstreamUncached checks that a cut-off document with -// nothing cached fails the request and is not cached. -func TestMetadataTruncatedUpstreamUncached(t *testing.T) { - u := newCutMetadataUpstream(t, truncPackument(50)) - u.set(truncPackument(50), true) - proxy, db, _, _ := setupTestProxy(t) - proxy.HTTPClient = u.Client() - proxy.CacheMetadata = true - handler := NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() - - if rec := serve(handler, "/pkg", nil); rec.Code != http.StatusBadGateway { - t.Errorf("status %d, want 502", rec.Code) - } - if entry, _ := db.GetMetadataCache("npm", "pkg"); entry != nil { - t.Error("cached the cut-off document") - } -} diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go deleted file mode 100644 index 340e3db..0000000 --- a/internal/handler/metadata_validators_test.go +++ /dev/null @@ -1,782 +0,0 @@ -package handler - -import ( - "fmt" - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/proxy/internal/database" -) - -// validatorUpstream serves one npm packument and one Composer document whose -// bodies a test can change, with a fixed Last-Modified. -type validatorUpstream struct { - *httptest.Server - mu sync.Mutex - npm string - composer string - lastModified time.Time -} - -func newValidatorUpstream(t *testing.T) *validatorUpstream { - t.Helper() - u := &validatorUpstream{ - npm: `{"name":"left-pad","versions":{"1.3.0":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"}}}}`, - composer: `{"packages":{"vendor/pkg":[{"version":"1.0.0","dist":{"type":"zip","url":"https://example.com/pkg-1.0.0.zip"}}]}}`, - lastModified: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), - } - u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - u.mu.Lock() - defer u.mu.Unlock() - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Last-Modified", u.lastModified.Format(http.TimeFormat)) - switch r.URL.Path { - case "/left-pad": - _, _ = w.Write([]byte(u.npm)) - case "/p2/vendor/pkg.json": - _, _ = w.Write([]byte(u.composer)) - default: - http.NotFound(w, r) - } - })) - t.Cleanup(u.Close) - return u -} - -func (u *validatorUpstream) set(npm, composer string) { - u.mu.Lock() - defer u.mu.Unlock() - u.npm, u.composer = npm, composer -} - -func (u *validatorUpstream) setLastModified(lastModified time.Time) { - u.mu.Lock() - defer u.mu.Unlock() - u.lastModified = lastModified -} - -type validatorCase struct { - ecosystem, name, path string - handler func(*Proxy, *validatorUpstream) http.Handler -} - -var validatorCases = []validatorCase{ - {"npm", "left-pad", "/left-pad", func(p *Proxy, u *validatorUpstream) http.Handler { - return NewNPMHandler(p, "http://proxy.example", u.URL).Routes() - }}, - {"composer", "vendor/pkg", "/p2/vendor/pkg.json", func(p *Proxy, u *validatorUpstream) http.Handler { - return NewComposerHandlerWithUpstreams(p, "http://proxy.example", u.URL, u.URL).Routes() - }}, -} - -func validatorProxy(t *testing.T, u *validatorUpstream, ttl time.Duration) *Proxy { - t.Helper() - proxy, _, _ := validatorProxyWithDB(t, u, ttl) - return proxy -} - -// validatorProxyWithDB returns a proxy that dates metadata by a clock the -// test moves, starting a second after the process start it models. -func validatorProxyWithDB(t *testing.T, u *validatorUpstream, ttl time.Duration) (*Proxy, *database.DB, *testClock) { - t.Helper() - proxy, db, _, _ := setupTestProxy(t) - proxy.HTTPClient = u.Client() - proxy.CacheMetadata = true - proxy.MetadataTTL = ttl - clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} - restartProxy(proxy, clock) - clock.advance(time.Second) - return proxy, db, clock -} - -// restartProxy gives proxy a new, empty rewrite cache and makes it date -// metadata by clock from the clock's current time on, as a restart of the -// process would. What the database holds survives. -func restartProxy(proxy *Proxy, clock *testClock) { - proxy.SetMetadataRewriteCacheSize(1 << 20) - proxy.now = clock.now - proxy.started = clock.now() -} - -// validatorReplica returns a second proxy process sharing proxy's database -// and storage, as another replica behind the same load balancer would, with -// its own rewrite cache and its own clock. -func validatorReplica(proxy *Proxy, clock *testClock) *Proxy { - replica := NewProxy(proxy.DB, proxy.Storage, proxy.Fetcher, proxy.Resolver, proxy.Logger) - replica.HTTPClient = proxy.HTTPClient - replica.CacheMetadata = proxy.CacheMetadata - replica.MetadataTTL = proxy.MetadataTTL - replica.SetMetadataRewriteCacheSize(1 << 20) - replica.now = clock.now - replica.started = proxy.started - return replica -} - -type testClock struct { - mu sync.Mutex - t time.Time -} - -func (c *testClock) now() time.Time { - c.mu.Lock() - defer c.mu.Unlock() - return c.t -} - -func (c *testClock) advance(d time.Duration) { - c.mu.Lock() - defer c.mu.Unlock() - c.t = c.t.Add(d) -} - -// npmV and composerV are upstream documents with a single version v. -func npmV(v string) string { - return `{"name":"left-pad","versions":{"` + v + `":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-` + v + `.tgz"}}}}` -} - -func composerV(v string) string { - return `{"packages":{"vendor/pkg":[{"version":"` + v + `","dist":{"type":"zip","url":"https://example.com/pkg-` + v + `.zip"}}]}}` -} - -// expireMetadata ages a metadata cache row past any TTL, so the next request -// refreshes it from upstream. -func expireMetadata(t *testing.T, db *database.DB, ecosystem, name string) { - t.Helper() - entry, err := db.GetMetadataCache(ecosystem, name) - if err != nil || entry == nil { - t.Fatalf("metadata row %s/%s = %v, %v", ecosystem, name, entry, err) - } - entry.FetchedAt.Time = time.Now().Add(-48 * time.Hour) - if err := db.UpsertMetadataCache(entry); err != nil { - t.Fatal(err) - } -} - -func serve(handler http.Handler, path string, header http.Header) *httptest.ResponseRecorder { - req := httptest.NewRequest(http.MethodGet, path, nil) - for k, v := range header { - req.Header[k] = v - } - rec := httptest.NewRecorder() - handler.ServeHTTP(rec, req) - return rec -} - -// TestRewrittenMetadataETag checks that rewritten metadata carries an ETag -// that clients can revalidate with, on the first fetch, on the rewrite fast -// path and when the TTL forces a revalidation. -func TestRewrittenMetadataETag(t *testing.T) { - for _, c := range validatorCases { - for _, ttl := range []time.Duration{time.Hour, 0} { - t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { - u := newValidatorUpstream(t) - handler := c.handler(validatorProxy(t, u, ttl), u) - - first := serve(handler, c.path, nil) - etag := first.Header().Get("ETag") - if first.Code != http.StatusOK || etag == "" { - t.Fatalf("first response: status %d, ETag %q; want 200 with an ETag", first.Code, etag) - } - for _, ifNoneMatch := range []string{etag, "W/" + etag, `"other", ` + etag} { - expectNotModified(t, serve(handler, c.path, http.Header{"If-None-Match": {ifNoneMatch}}), etag) - } - if rec := serve(handler, c.path, nil); rec.Header().Get("ETag") != etag || rec.Body.String() != first.Body.String() { - t.Errorf("repeat response: ETag %q, want %q, and the same body", rec.Header().Get("ETag"), etag) - } - }) - } - } -} - -func expectNotModified(t *testing.T, rec *httptest.ResponseRecorder, etag string) { - t.Helper() - if rec.Code != http.StatusNotModified || rec.Body.Len() != 0 { - t.Errorf("status %d with %d bytes, want an empty 304", rec.Code, rec.Body.Len()) - } - if got := rec.Header().Get("ETag"); got != etag { - t.Errorf("304 ETag = %q, want %q", got, etag) - } -} - -// TestRewrittenMetadataETagFollowsUpstream checks that a new upstream -// document gets a new ETag, so a client's old copy no longer matches. -func TestRewrittenMetadataETagFollowsUpstream(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := newValidatorUpstream(t) - handler := c.handler(validatorProxy(t, u, 0), u) - etag := serve(handler, c.path, nil).Header().Get("ETag") - - u.set( - `{"name":"left-pad","versions":{"1.3.1":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.1.tgz"}}}}`, - `{"packages":{"vendor/pkg":[{"version":"1.0.1","dist":{"type":"zip","url":"https://example.com/pkg-1.0.1.zip"}}]}}`, - ) - rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}) - if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { - t.Errorf("status %d, ETag %q; want 200 with a new ETag", rec.Code, rec.Header().Get("ETag")) - } - }) - } -} - -// TestRewrittenMetadataLastModified checks the validator Composer revalidates -// with. It is when this process started serving the rewrite, not upstream's -// date: the rewrite also depends on configuration read at startup, so the -// same upstream document can be rewritten differently after a restart. It -// stays the same while the rewrite does. -func TestRewrittenMetadataLastModified(t *testing.T) { - for _, c := range validatorCases { - for _, ttl := range []time.Duration{time.Hour, 0} { - t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { - u := newValidatorUpstream(t) - proxy, _, clock := validatorProxyWithDB(t, u, ttl) - handler := c.handler(proxy, u) - - clock.advance(400 * time.Millisecond) - first := serve(handler, c.path, nil) - lastModified, err := http.ParseTime(first.Header().Get("Last-Modified")) - if err != nil { - t.Fatalf("Last-Modified %q: %v", first.Header().Get("Last-Modified"), err) - } - if want := clock.now().Truncate(time.Second); !lastModified.Equal(want) { - t.Errorf("Last-Modified %v, want %v, the second of the rewrite", lastModified, want) - } - clock.advance(time.Hour) - - current := serve(handler, c.path, http.Header{"If-Modified-Since": {first.Header().Get("Last-Modified")}}) - if current.Code != http.StatusNotModified || current.Body.Len() != 0 { - t.Errorf("If-Modified-Since Last-Modified: status %d with %d bytes, want an empty 304", current.Code, current.Body.Len()) - } - - older := lastModified.Add(-time.Hour).Format(http.TimeFormat) - if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {older}}); rec.Code != http.StatusOK { - t.Errorf("If-Modified-Since an hour earlier: status %d, want 200", rec.Code) - } - - // If-None-Match takes precedence over If-Modified-Since. - rec := serve(handler, c.path, http.Header{ - "If-None-Match": {`"other"`}, - "If-Modified-Since": {first.Header().Get("Last-Modified")}, - }) - if rec.Code != http.StatusOK { - t.Errorf("mismatched If-None-Match with a current If-Modified-Since: status %d, want 200", rec.Code) - } - }) - } - } -} - -// TestRewrittenMetadataLastModifiedFollowsRefresh is a regression test for a -// false 304. Upstream dates both documents before this process started, the -// cached one and the newer one a refresh fetches, and neither may give the -// newer document a Last-Modified a client holding the older one already has: -// If-Modified-Since from the first response must fetch the new body. -func TestRewrittenMetadataLastModifiedFollowsRefresh(t *testing.T) { - for _, c := range validatorCases { - for _, ttl := range []time.Duration{time.Hour, 0} { - t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { - u := newValidatorUpstream(t) - proxy, db, clock := validatorProxyWithDB(t, u, ttl) - handler := c.handler(proxy, u) - - first := serve(handler, c.path, nil) - firstModified := first.Header().Get("Last-Modified") - if first.Code != http.StatusOK || firstModified == "" { - t.Fatalf("first response: status %d, Last-Modified %q; want 200 with a Last-Modified", first.Code, firstModified) - } - - clock.advance(time.Minute) - u.set(npmV("1.3.1"), composerV("1.0.1")) - u.setLastModified(u.lastModified.Add(24 * time.Hour)) - expireMetadata(t, db, c.ecosystem, c.name) - - rec := serve(handler, c.path, http.Header{"If-Modified-Since": {firstModified}}) - if rec.Code != http.StatusOK || rec.Body.String() == first.Body.String() { - t.Fatalf("If-Modified-Since from before the refresh: status %d, body changed %t; want 200 with the new body", - rec.Code, rec.Body.String() != first.Body.String()) - } - was, _ := http.ParseTime(firstModified) - now, err := http.ParseTime(rec.Header().Get("Last-Modified")) - if err != nil || !now.After(was) { - t.Errorf("Last-Modified after the refresh = %q, want later than %q", rec.Header().Get("Last-Modified"), firstModified) - } - if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {rec.Header().Get("Last-Modified")}}); rec.Code != http.StatusNotModified { - t.Errorf("If-Modified-Since the new Last-Modified: status %d, want 304", rec.Code) - } - }) - } - } -} - -// TestRewrittenMetadataLastModifiedReturningDocument is a regression test -// for a false 304. Upstream serves A, then B, then A again; A's cached -// rewrite must not bring back the date it was first served with, which is -// earlier than B's, or a client holding B gets 304 for A. -func TestRewrittenMetadataLastModifiedReturningDocument(t *testing.T) { - for _, c := range validatorCases { - for _, ttl := range []time.Duration{time.Hour, 0} { - t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { - u := newValidatorUpstream(t) - proxy, db, clock := validatorProxyWithDB(t, u, ttl) - handler := c.handler(proxy, u) - change := func(v string) { - clock.advance(10 * time.Second) - u.set(npmV(v), composerV(v)) - if ttl > 0 { - expireMetadata(t, db, c.ecosystem, c.name) - } - } - - u.set(npmV("1.0.0"), composerV("1.0.0")) - a := serve(handler, c.path, nil) - change("2.0.0") - b := serve(handler, c.path, nil) - if b.Body.String() == a.Body.String() || b.Header().Get("Last-Modified") == "" { - t.Fatalf("B: Last-Modified %q, body changed %t; want a new body with a date", b.Header().Get("Last-Modified"), b.Body.String() != a.Body.String()) - } - change("1.0.0") - - rec := serve(handler, c.path, http.Header{"If-Modified-Since": {b.Header().Get("Last-Modified")}}) - if rec.Code != http.StatusOK || rec.Body.String() != a.Body.String() { - t.Fatalf("A again, If-Modified-Since B's date: status %d, body is A %t; want 200 with A", rec.Code, rec.Body.String() == a.Body.String()) - } - was, _ := http.ParseTime(b.Header().Get("Last-Modified")) - if now, err := http.ParseTime(rec.Header().Get("Last-Modified")); err != nil || !now.After(was) { - t.Errorf("A again: Last-Modified %q, want later than B's %q", rec.Header().Get("Last-Modified"), b.Header().Get("Last-Modified")) - } - }) - } - } -} - -// TestRewrittenMetadataLastModifiedSameSecond checks that a document that -// replaces another within the second the other was dated gets no date of -// its own until a later second, rather than one ahead of the clock. The -// clock-time date it would share is one a client may already hold. -func TestRewrittenMetadataLastModifiedSameSecond(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := newValidatorUpstream(t) - proxy, _, clock := validatorProxyWithDB(t, u, 0) - handler := c.handler(proxy, u) - - a := serve(handler, c.path, nil) - aModified := a.Header().Get("Last-Modified") - u.set(npmV("2.0.0"), composerV("2.0.0")) - for i, header := range []http.Header{nil, {"If-Modified-Since": {aModified}}} { - rec := serve(handler, c.path, header) - if rec.Code != http.StatusOK || rec.Body.String() == a.Body.String() { - t.Fatalf("request %d for B: status %d; want 200 with B", i, rec.Code) - } - if lm := rec.Header().Get("Last-Modified"); lm != "" { - t.Errorf("request %d for B in A's second: Last-Modified %q, want none", i, lm) - } - } - - clock.advance(time.Second) - rec := serve(handler, c.path, nil) - lastModified, err := http.ParseTime(rec.Header().Get("Last-Modified")) - if want := clock.now().Truncate(time.Second); err != nil || !lastModified.Equal(want) { - t.Errorf("B a second later: Last-Modified %q, want %v", rec.Header().Get("Last-Modified"), want) - } - }) - } -} - -// TestRewrittenMetadataLastModifiedAcrossRestart is a regression test for a -// false 304 after a restart. Dates are never ahead of the clock, and the -// restarted process dates nothing in the second it starts, the last in which -// the previous process may have dated a document. -func TestRewrittenMetadataLastModifiedAcrossRestart(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := newValidatorUpstream(t) - proxy, _, clock := validatorProxyWithDB(t, u, 0) - handler := c.handler(proxy, u) - - // Two documents in one second; the client keeps the latest date. - latest := serve(handler, c.path, nil).Header().Get("Last-Modified") - u.set(npmV("2.0.0"), composerV("2.0.0")) - if lm := serve(handler, c.path, nil).Header().Get("Last-Modified"); lm != "" { - latest = lm - } - if held, _ := http.ParseTime(latest); held.After(clock.now()) { - t.Errorf("Last-Modified %q is ahead of the clock at %v", latest, clock.now()) - } - - restartProxy(proxy, clock) - u.set(npmV("3.0.0"), composerV("3.0.0")) - rec := serve(handler, c.path, http.Header{"If-Modified-Since": {latest}}) - if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "3.0.0") { - t.Fatalf("changed document after a restart, If-Modified-Since %q: status %d; want 200 with it", latest, rec.Code) - } - if lm := rec.Header().Get("Last-Modified"); lm != "" { - t.Errorf("Last-Modified %q in the second the process started, want none", lm) - } - - clock.advance(time.Second) - rec = serve(handler, c.path, http.Header{"If-Modified-Since": {latest}}) - if rec.Code != http.StatusOK || rec.Header().Get("Last-Modified") == "" { - t.Errorf("a second after the restart: status %d, Last-Modified %q; want 200 with a date", rec.Code, rec.Header().Get("Last-Modified")) - } - }) - } -} - -// TestRewrittenMetadataValidatorsWithCooldown checks that with cooldown on, -// rewritten metadata from the rewrite cache has an ETag and a Last-Modified, -// so clients that only send If-Modified-Since, such as Composer, get 304. -func TestRewrittenMetadataValidatorsWithCooldown(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := newValidatorUpstream(t) - proxy := validatorProxy(t, u, time.Hour) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - handler := c.handler(proxy, u) - - first := serve(handler, c.path, nil) - etag := first.Header().Get("ETag") - if etag == "" { - t.Fatal("no ETag with cooldown on") - } - lm := first.Header().Get("Last-Modified") - if lm == "" { - t.Fatal("no Last-Modified with cooldown on") - } - if rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}); rec.Code != http.StatusNotModified { - t.Errorf("If-None-Match: status %d, want 304", rec.Code) - } - if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {lm}}); rec.Code != http.StatusNotModified { - t.Errorf("If-Modified-Since: status %d, want 304", rec.Code) - } - }) - } -} - -// TestRewrittenMetadataWithCooldownIsGzipped checks that with cooldown on, -// clients accepting gzip get the cached compressed copy. -func TestRewrittenMetadataWithCooldownIsGzipped(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := newValidatorUpstream(t) - compressibleDocuments(u) - proxy := validatorProxy(t, u, time.Hour) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - handler := c.handler(proxy, u) - for range 2 { - if rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}}); rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "gzip" { - t.Errorf("status %d, Content-Encoding %q; want 200 gzip", rec.Code, rec.Header().Get("Content-Encoding")) - } - } - }) - } -} - -// TestUnrewrittenMetadataHasNoValidators checks that metadata proxied as it -// is, because rewriting it failed, does not claim the rewrite's validators. -func TestUnrewrittenMetadataHasNoValidators(t *testing.T) { - u := newValidatorUpstream(t) - u.set(`{"name":"left-pad","versions":`, `{"packages":`) - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - rec := serve(c.handler(validatorProxy(t, u, time.Hour), u), c.path, nil) - if rec.Code != http.StatusOK { - t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) - } - if etag, lm := rec.Header().Get("ETag"), rec.Header().Get("Last-Modified"); etag != "" || lm != "" { - t.Errorf("ETag %q, Last-Modified %q; want neither on unrewritten metadata", etag, lm) - } - }) - } -} - -func TestRewriteCacheETag(t *testing.T) { - c := newRewriteCache(1 << 20) - exclaim := func(b []byte) ([]byte, error) { return []byte(string(b) + "!"), nil } - key := rewriteCacheKey("npm", "http://proxy", "demo", []byte("doc")) - - m, err := c.rewrite(t.Context(), key, []byte("doc"), nil, exclaim) - if err != nil || string(m.content()) != "doc!" { - t.Fatalf("rewrite = %q, %v", m.content(), err) - } - if want := metadataETag([]byte("doc!")); m.etag != want { - t.Errorf("rewrite ETag = %q, want %q", m.etag, want) - } - if cached, ok := c.get(key); !ok || cached.metadataValidators != m.metadataValidators { - t.Errorf("get validators = %+v, %v; want %+v", cached.metadataValidators, ok, m.metadataValidators) - } - if metadataETag([]byte("other")) == m.etag { - t.Error("different output has the same ETag") - } -} - -// TestRewrittenMetadataLastModifiedWithoutRewriteCache checks that dates do -// not depend on the rewrite cache: the database records them. -func TestRewrittenMetadataLastModifiedWithoutRewriteCache(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := newValidatorUpstream(t) - proxy, _, clock := validatorProxyWithDB(t, u, time.Hour) - proxy.SetMetadataRewriteCacheSize(0) - handler := c.handler(proxy, u) - - first := serve(handler, c.path, nil) - if first.Code != http.StatusOK || first.Header().Get("ETag") == "" { - t.Fatalf("status %d, ETag %q; want 200 with an ETag", first.Code, first.Header().Get("ETag")) - } - lastModified, err := http.ParseTime(first.Header().Get("Last-Modified")) - if want := clock.now().Truncate(time.Second); err != nil || !lastModified.Equal(want) { - t.Errorf("Last-Modified %q without the rewrite cache, want %v", first.Header().Get("Last-Modified"), want) - } - clock.advance(time.Minute) - if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {first.Header().Get("Last-Modified")}}); rec.Code != http.StatusNotModified { - t.Errorf("If-Modified-Since Last-Modified: status %d, want 304", rec.Code) - } - }) - } -} - -// TestRewrittenMetadataLastModifiedAcrossReplicas is a regression test for a -// false 304 between replicas sharing a database and storage. One replica -// dates a document, upstream changes in that second, and a client holding -// the date asks another replica: the date must not suppress the new body. -// Once a second has passed, both replicas date the new document alike. -func TestRewrittenMetadataLastModifiedAcrossReplicas(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := newValidatorUpstream(t) - a, _, clock := validatorProxyWithDB(t, u, 0) - b := validatorReplica(a, clock) - replicas := map[string]http.Handler{"A": c.handler(a, u), "B": c.handler(b, u)} - - first := serve(replicas["A"], c.path, nil) - held := first.Header().Get("Last-Modified") - if first.Code != http.StatusOK || held == "" { - t.Fatalf("replica A: status %d, Last-Modified %q; want 200 with a date", first.Code, held) - } - u.set(npmV("2.0.0"), composerV("2.0.0")) - for _, name := range []string{"B", "A"} { - expectChangedUndated(t, name, serve(replicas[name], c.path, http.Header{"If-Modified-Since": {held}}), first) - } - - clock.advance(time.Second) - shared := expectSharedDate(t, replicas, c.path, held) - for name, handler := range replicas { - if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {shared}}); rec.Code != http.StatusNotModified { - t.Errorf("replica %s, If-Modified-Since the shared date: status %d, want 304", name, rec.Code) - } - if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {held}}); rec.Code != http.StatusOK { - t.Errorf("replica %s, If-Modified-Since the old date: status %d, want 200", name, rec.Code) - } - } - }) - } -} - -// expectChangedUndated checks that a replica answered a conditional request -// with a body and ETag other than first's, and no date: the second first was -// dated in may not date another representation. -func expectChangedUndated(t *testing.T, replica string, rec *httptest.ResponseRecorder, first *httptest.ResponseRecorder) { - t.Helper() - if rec.Code != http.StatusOK || rec.Body.String() == first.Body.String() { - t.Fatalf("replica %s after upstream changed: status %d; want 200 with the new body", replica, rec.Code) - } - if rec.Header().Get("ETag") == first.Header().Get("ETag") { - t.Errorf("replica %s: the new body has the old ETag", replica) - } - if lm := rec.Header().Get("Last-Modified"); lm != "" { - t.Errorf("replica %s: Last-Modified %q in the second the old body was dated, want none", replica, lm) - } -} - -// expectSharedDate checks that every replica serves the same Last-Modified, -// later than held, and returns it. -func expectSharedDate(t *testing.T, replicas map[string]http.Handler, path, held string) string { - t.Helper() - was, _ := http.ParseTime(held) - shared := "" - for name, handler := range replicas { - lm := serve(handler, path, nil).Header().Get("Last-Modified") - if now, err := http.ParseTime(lm); err != nil || !now.After(was) { - t.Fatalf("replica %s: Last-Modified %q, want later than %q", name, lm, held) - } - if shared != "" && lm != shared { - t.Fatalf("replica %s: Last-Modified %q, another replica serves %q", name, lm, shared) - } - shared = lm - } - return shared -} - -// TestRewrittenMetadataLastModifiedFromBeforeTheFix checks that dates clients -// hold from before dates were shared cannot suppress a body. Those were never -// later than the clock of the process that gave them, which stopped before -// this one started, so no date is given in the second this process started -// and every later one is later than any of them. -func TestRewrittenMetadataLastModifiedFromBeforeTheFix(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := newValidatorUpstream(t) - a, _, clock := validatorProxyWithDB(t, u, 0) - restartProxy(a, clock) - b := validatorReplica(a, clock) - held := clock.now().Truncate(time.Second).Format(http.TimeFormat) - - for name, handler := range map[string]http.Handler{"A": c.handler(a, u), "B": c.handler(b, u)} { - rec := serve(handler, c.path, http.Header{"If-Modified-Since": {held}}) - if rec.Code != http.StatusOK || rec.Header().Get("Last-Modified") != "" { - t.Errorf("replica %s in its first second, If-Modified-Since %q: status %d, Last-Modified %q; want 200 without a date", - name, held, rec.Code, rec.Header().Get("Last-Modified")) - } - } - - clock.advance(time.Second) - for name, handler := range map[string]http.Handler{"A": c.handler(a, u), "B": c.handler(b, u)} { - rec := serve(handler, c.path, http.Header{"If-Modified-Since": {held}}) - was, _ := http.ParseTime(held) - now, err := http.ParseTime(rec.Header().Get("Last-Modified")) - if rec.Code != http.StatusOK || err != nil || !now.After(was) { - t.Errorf("replica %s a second later, If-Modified-Since %q: status %d, Last-Modified %q; want 200 with a later date", - name, held, rec.Code, rec.Header().Get("Last-Modified")) - } - } - }) - } -} - -// TestRewrittenMetadataLastModifiedWithClockSkew checks that a replica whose -// clock is behind gives no date ahead of its own clock, neither for a new -// document nor for one another replica dated. -func TestRewrittenMetadataLastModifiedWithClockSkew(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := newValidatorUpstream(t) - a, _, clock := validatorProxyWithDB(t, u, 0) - behind := &testClock{t: clock.now().Add(-2 * time.Second)} - b := validatorReplica(a, behind) - handlerA, handlerB := c.handler(a, u), c.handler(b, u) - - held := serve(handlerA, c.path, nil).Header().Get("Last-Modified") - u.set(npmV("2.0.0"), composerV("2.0.0")) - if rec := serve(handlerB, c.path, http.Header{"If-Modified-Since": {held}}); rec.Code != http.StatusOK || rec.Header().Get("Last-Modified") != "" { - t.Fatalf("lagging replica, new document: status %d, Last-Modified %q; want 200 without a date", rec.Code, rec.Header().Get("Last-Modified")) - } - - clock.advance(time.Second) - dated := serve(handlerA, c.path, nil).Header().Get("Last-Modified") - if dated == "" { - t.Fatal("replica A gave no date a second later") - } - if lm := serve(handlerB, c.path, nil).Header().Get("Last-Modified"); lm != "" { - t.Errorf("lagging replica: Last-Modified %q ahead of its clock at %v, want none", lm, behind.now()) - } - behind.advance(3 * time.Second) - if lm := serve(handlerB, c.path, nil).Header().Get("Last-Modified"); lm != dated { - t.Errorf("lagging replica once its clock caught up: Last-Modified %q, want %q", lm, dated) - } - }) - } -} - -// cooldownUpstream sets documents with one version published long ago and -// one published two hours ago, so a cooldown of a day hides the recent one -// and a cooldown of an hour keeps it. -func cooldownUpstream(t *testing.T) *validatorUpstream { - t.Helper() - recent := time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339) - u := newValidatorUpstream(t) - u.set( - `{"name":"left-pad","dist-tags":{"latest":"1.4.0"},"versions":{`+ - `"1.3.0":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"}},`+ - `"1.4.0":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.4.0.tgz"}}},`+ - `"time":{"1.3.0":"2016-01-01T00:00:00Z","1.4.0":"`+recent+`"}}`, - `{"packages":{"vendor/pkg":[`+ - `{"version":"1.0.0","time":"2016-01-01T00:00:00Z","dist":{"type":"zip","url":"https://example.com/pkg-1.0.0.zip"}},`+ - `{"version":"1.1.0","time":"`+recent+`","dist":{"type":"zip","url":"https://example.com/pkg-1.1.0.zip"}}]}}`, - ) - return u -} - -// TestRewrittenMetadataETagWithCooldownFollowsKeptVersions checks that with -// cooldown on, the ETag changes when a version leaves cooldown although the -// upstream document stays the same, once the cached rewrite has expired, and -// comes back when the same versions do. -func TestRewrittenMetadataETagWithCooldownFollowsKeptVersions(t *testing.T) { - for _, c := range validatorCases { - t.Run(c.ecosystem, func(t *testing.T) { - u := cooldownUpstream(t) - proxy, _, clock := validatorProxyWithDB(t, u, time.Hour) - handler := c.handler(proxy, u) - - proxy.Cooldown = &cooldown.Config{Default: "1d"} - etag := serve(handler, c.path, nil).Header().Get("ETag") - - proxy.Cooldown = &cooldown.Config{Default: "1h"} - if rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}); rec.Code != http.StatusNotModified { - t.Errorf("within the rewrite's lifetime: status %d, want 304 from the cached rewrite", rec.Code) - } - clock.advance(maxCooldownRewriteTTL) - rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}) - if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { - t.Errorf("after a version left cooldown: status %d, ETag %q; want 200 with a new ETag", rec.Code, rec.Header().Get("ETag")) - } - - proxy.Cooldown = &cooldown.Config{Default: "1d"} - clock.advance(maxCooldownRewriteTTL) - expectNotModified(t, serve(handler, c.path, http.Header{"If-None-Match": {etag}}), etag) - }) - } -} - -// TestRewrittenMetadataETagWithCooldownComesFromInputs checks that with -// cooldown on and no rewrite cache, the ETag is derived from what fixes the -// rewrite rather than from the rewritten body, when the metadata cache knows -// the upstream digest, and falls back to the body when it does not. -func TestRewrittenMetadataETagWithCooldownComesFromInputs(t *testing.T) { - for _, c := range validatorCases { - for _, cacheMetadata := range []bool{true, false} { - t.Run(fmt.Sprintf("%s/cacheMetadata=%t", c.ecosystem, cacheMetadata), func(t *testing.T) { - u := cooldownUpstream(t) - proxy := validatorProxy(t, u, time.Hour) - proxy.CacheMetadata = cacheMetadata - proxy.SetMetadataRewriteCacheSize(0) - proxy.Cooldown = &cooldown.Config{Default: "1d"} - handler := c.handler(proxy, u) - - rec := serve(handler, c.path, nil) - bodyETag := metadataETag(rec.Body.Bytes()) - if got := rec.Header().Get("ETag"); (got == bodyETag) == cacheMetadata { - t.Errorf("ETag %q, body hash %q; want them to differ only with a known digest", got, bodyETag) - } - etag := rec.Header().Get("ETag") - expectNotModified(t, serve(handler, c.path, http.Header{"If-None-Match": {etag}}), etag) - }) - } - } -} - -func TestKeptVersionsETag(t *testing.T) { - base := keptVersionsETag("sha256:abc", "http://proxy", "left-pad", []string{"1.0.0", "1.1.0"}) - if got := keptVersionsETag("sha256:abc", "http://proxy", "left-pad", []string{"1.1.0", "1.0.0"}); got != base { - t.Errorf("kept versions in another order: ETag %q, want %q", got, base) - } - for name, etag := range map[string]string{ - "digest": keptVersionsETag("sha256:abd", "http://proxy", "left-pad", []string{"1.0.0", "1.1.0"}), - "proxy URL": keptVersionsETag("sha256:abc", "http://other", "left-pad", []string{"1.0.0", "1.1.0"}), - "name": keptVersionsETag("sha256:abc", "http://proxy", "right-pad", []string{"1.0.0", "1.1.0"}), - "kept versions": keptVersionsETag("sha256:abc", "http://proxy", "left-pad", []string{"1.0.0"}), - "field bounds": keptVersionsETag("sha256:abc", "http://proxy", "left-pad", []string{"1.0.0\x001.1.0"}), - } { - if etag == base { - t.Errorf("a different %s gives the same ETag", name) - } - } - if !strings.HasPrefix(base, `"`) || !strings.HasSuffix(base, `"`) { - t.Errorf("ETag %q is not a quoted strong validator", base) - } -} diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go deleted file mode 100644 index cc61928..0000000 --- a/internal/handler/notfound_ecosystems_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package handler - -import ( - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/git-pkgs/registries/fetch" -) - -func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { - tests := []struct { - name string - path string - handler func(p *Proxy) http.Handler - }{ - {"debian", "/pool/main/n/nginx/nginx_1.18.0-6_amd64.deb", - func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://localhost", "", nil).Routes() }}, - {"rpm", "/releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm", - func(p *Proxy) http.Handler { return NewRPMHandler(p, "http://localhost").Routes() }}, - {"apk", "/alpine/v3.22/main/x86_64/busybox-1.37.0-r12.apk", - func(p *Proxy) http.Handler { return NewAPKHandler(p, "http://localhost", nil).Routes() }}, - {"arch", "/archlinux/core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst", - func(p *Proxy) http.Handler { return NewArchHandler(p, "http://localhost", nil).Routes() }}, - {"freebsd", "/freebsd/FreeBSD:14:amd64/latest/All/Hashed/zsh-5.9_5~2$3rwh1rgp.pkg", - func(p *Proxy) http.Handler { return NewFreeBSDHandler(p, "http://localhost", nil).Routes() }}, - {"nuget", "/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", - func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://localhost").Routes() }}, - {"pypi", "/packages/packages/ab/cd/ef0123456789/requests-2.31.0-py3-none-any.whl", - func(p *Proxy) http.Handler { - return NewPyPIHandlerWithUpstreams(p, "http://localhost", "", "").Routes() - }}, - {"cran", "/src/contrib/ggplot2_3.4.4.tar.gz", - func(p *Proxy) http.Handler { return NewCRANHandler(p, "http://localhost").Routes() }}, - {"conda", "/conda-forge/linux-64/numpy-1.26.0-py311_0.tar.bz2", - func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://localhost").Routes() }}, - {"conan", "/v2/conans/zlib/1.3.1/_/_/revisions/0/files/conan_sources.tgz", - func(p *Proxy) http.Handler { return NewConanHandler(p, "http://localhost").Routes() }}, - {"gem", "/gems/rails-7.1.0.gem", - func(p *Proxy) http.Handler { return NewGemHandler(p, "http://localhost").Routes() }}, - {"hex", "/tarballs/phoenix-1.7.10.tar", - func(p *Proxy) http.Handler { return NewHexHandler(p, "http://localhost").Routes() }}, - {"chef", "/api/v1/cookbooks/apt/versions/7.5.22/download", - func(p *Proxy) http.Handler { return NewChefHandler(p, "http://localhost", "http://localhost").Routes() }}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.fetchErr = fetch.ErrNotFound - - srv := httptest.NewServer(tt.handler(proxy)) - defer srv.Close() - - resp, err := http.Get(srv.URL + tt.path) - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusNotFound { - t.Errorf("want 404 for missing upstream artifact, got %d", resp.StatusCode) - } - }) - } -} - -func TestJuliaPackageUpstreamNotFoundReturns404(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.fetchErr = fetch.ErrNotFound - - dead := httptest.NewServer(http.NotFoundHandler()) - defer dead.Close() - - h := NewJuliaHandler(proxy, "http://localhost") - h.upstreamURL = dead.URL - - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + - "/package/7876af07-990d-54b4-ab0e-23690620f79a/0123456789abcdef0123456789abcdef01234567") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusNotFound { - t.Errorf("want 404 for missing upstream package, got %d", resp.StatusCode) - } -} - -func TestComposerDownloadUpstreamNotFoundReturns404(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.fetchErr = fetch.ErrNotFound - - meta := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/p2/monolog/monolog.json" { - _, _ = w.Write([]byte(`{ - "packages": { - "monolog/monolog": [ - {"version": "2.9.1", "dist": {"url": "https://example.com/monolog-2.9.1.zip", "type": "zip"}} - ] - } - }`)) - return - } - http.NotFound(w, r) - })) - defer meta.Close() - - h := &ComposerHandler{proxy: proxy, repoURL: meta.URL, proxyURL: "http://localhost"} - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/files/monolog/monolog/2.9.1/monolog-2.9.1.zip") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusNotFound { - t.Errorf("want 404 for missing upstream dist, got %d", resp.StatusCode) - } -} - -func TestContainerBlobUpstreamNotFoundReturns404(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.fetchErr = fetch.ErrNotFound - - h := &ContainerHandler{ - proxy: proxy, - registryURL: "https://registry-1.docker.io", - proxyURL: "http://localhost:8080", - } - - req := httptest.NewRequest(http.MethodGet, - "/library/nginx/blobs/sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusNotFound { - t.Errorf("want 404 for missing upstream blob, got %d; body: %s", w.Code, w.Body.String()) - } - if !strings.Contains(w.Body.String(), "BLOB_UNKNOWN") { - t.Errorf("want BLOB_UNKNOWN error code in body, got: %s", w.Body.String()) - } -} diff --git a/internal/handler/notfound_test.go b/internal/handler/notfound_test.go deleted file mode 100644 index 9ea38ac..0000000 --- a/internal/handler/notfound_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/git-pkgs/registries/fetch" -) - -func TestErrUpstreamNotFoundWrapsFetchErrNotFound(t *testing.T) { - if !errors.Is(ErrUpstreamNotFound, fetch.ErrNotFound) { - t.Fatal("ErrUpstreamNotFound does not wrap fetch.ErrNotFound") - } -} - -func TestGetOrFetchArtifactFromURL_NotFound(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.fetchErr = fetch.ErrNotFound - - _, err := proxy.GetOrFetchArtifactFromURL(context.Background(), - "maven", "org.example:missing", "1.0", "missing-1.0.jar", - "http://upstream.test/org/example/missing/1.0/missing-1.0.jar") - - if !errors.Is(err, ErrUpstreamNotFound) { - t.Fatalf("want ErrUpstreamNotFound, got %v", err) - } -} - -func TestMavenHandler_UpstreamNotFoundReturns404(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.fetchErr = fetch.ErrNotFound - - h := NewMavenHandler(proxy, "http://localhost", "http://upstream.test", "http://portal.test") - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/org/example/missing/1.0/missing-1.0.jar") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusNotFound { - t.Errorf("want 404 for missing upstream artifact, got %d", resp.StatusCode) - } -} - -func TestMavenHandler_PluginPortalFallback(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.fetchErrByURL = map[string]error{ - "http://upstream.test/org/example/plugin/1.0/plugin-1.0.jar": fetch.ErrNotFound, - } - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("portal artifact")), - ContentType: "application/java-archive", - } - - h := NewMavenHandler(proxy, "http://localhost", "http://upstream.test", "http://portal.test") - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/org/example/plugin/1.0/plugin-1.0.jar") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - t.Fatalf("want 200 via plugin portal fallback, got %d", resp.StatusCode) - } - body, _ := io.ReadAll(resp.Body) - if string(body) != "portal artifact" { - t.Errorf("want portal artifact body, got %q", body) - } - if fetcher.fetchedURL != "http://portal.test/org/example/plugin/1.0/plugin-1.0.jar" { - t.Errorf("fallback did not hit plugin portal, last URL: %s", fetcher.fetchedURL) - } -} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 7eef424..0585eda 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -1,136 +1,49 @@ package handler import ( - "bytes" - "context" + "encoding/json" "errors" "fmt" - "io" "net/http" "net/url" + "sort" "strings" - "sync" "time" - "github.com/git-pkgs/integrity" - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/jsonscan" - "github.com/git-pkgs/proxy/internal/metadata/npm" - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/purl" ) const ( npmUpstream = "https://registry.npmjs.org" - npmAcceptDefault = "application/vnd.npm.install-v1+json;q=1.0, application/json;q=0.8" + npmAbbreviatedCT = "application/vnd.npm.install-v1+json" scopedParts = 2 // scope + name in scoped packages - - // npmSecurityPrefix covers the audit endpoints: /-/npm/v1/security/audits, - // .../audits/quick and .../advisories/bulk. - npmSecurityPrefix = "/-/npm/v1/security/" - - // npmKeysPath serves the registry signing keys `npm audit signatures` reads. - npmKeysPath = "/-/npm/v1/keys" - - // npmAttestationsPrefix serves a version's provenance and publish - // attestations, as /-/npm/v1/attestations/{name}@{version}. - npmAttestationsPrefix = "/-/npm/v1/attestations/" - - // npmSearchPath is what `npm search` queries, with the terms in the query - // string. npmPingPath is what `npm ping` checks. - npmSearchPath = "/-/v1/search" - npmPingPath = "/-/ping" - - // npmSecurityMaxBody caps the audit payload. Buffering it lets an oversized - // body be refused before the upstream request starts. - npmSecurityMaxBody = 16 << 20 ) -// npm tarballs are stored under pkg:npm/{name} as purl builds it, so package -// rules use the default key mapping. -func init() { - retention.Default.Register(retention.Spec{Key: "npm", CanonicalPackage: retention.DefaultCanonical("npm")}) -} - // NPMHandler handles npm registry protocol requests. type NPMHandler struct { proxy *Proxy upstreamURL string proxyURL string // URL where this proxy is hosted - routes packageRoutes } // NewNPMHandler creates a new npm protocol handler. -func NewNPMHandler(proxy *Proxy, proxyURL, upstreamURL string) *NPMHandler { - if strings.TrimSpace(upstreamURL) == "" { - upstreamURL = npmUpstream - } - +func NewNPMHandler(proxy *Proxy, proxyURL string) *NPMHandler { return &NPMHandler{ proxy: proxy, - upstreamURL: strings.TrimSuffix(upstreamURL, "/"), + upstreamURL: npmUpstream, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } -// WithPackageRoutes sends packages matching a pattern to a dedicated npm -// registry instead of the default one. See packageRoutes. -func (h *NPMHandler) WithPackageRoutes(routes map[string]string) *NPMHandler { - h.routes = newPackageRoutes(routes) - return h -} - // Routes returns the HTTP handler for npm requests. // Mount this at /npm on your router. func (h *NPMHandler) Routes() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // The /-/npm/v1 endpoints share the /-/ prefix with tarball paths, so - // they are routed before the tarball dispatch below. Audits are POSTs, - // so they also precede the GET-only gate. - if strings.HasPrefix(r.URL.Path, npmSecurityPrefix) { - h.handleSecurity(w, r) - return - } - - // pnpm 12 checks packuments with HEAD before fetching them, so - // packuments also answer HEAD; net/http leaves out the body. - if r.Method == http.MethodHead && isPackumentPath(r.URL.Path) { - h.handlePackageMetadata(w, r) - return - } - if r.Method != http.MethodGet { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } - if strings.HasPrefix(r.URL.Path, npmAttestationsPrefix) { - h.handleAttestations(w, r) - return - } - - switch r.URL.Path { - case npmKeysPath: - h.proxy.ProxyUpstream(w, r, h.upstreamURL+npmKeysPath, - []string{headerAccept, headerAcceptEncoding}) - return - case npmSearchPath, npmPingPath: - // Every query has its own answer, so these are relayed uncached - // and keep their query string. - upstreamURL := h.upstreamURL + r.URL.Path - if r.URL.RawQuery != "" { - upstreamURL += "?" + r.URL.RawQuery - } - h.proxy.ProxyUpstream(w, r, upstreamURL, - []string{headerAccept, headerAcceptEncoding}) - return - } - - if strings.HasPrefix(r.URL.Path, npmPackagePrefix) && strings.HasSuffix(r.URL.Path, npmDistTagsSuffix) { - h.handleDistTags(w, r) - return - } - path := strings.TrimPrefix(r.URL.Path, "/") // Check if this is a tarball download (contains /-/) @@ -144,148 +57,29 @@ func (h *NPMHandler) Routes() http.Handler { }) } -// isPackumentPath reports whether a request path names package metadata -// rather than a tarball or a registry endpoint under /-/. -func isPackumentPath(urlPath string) bool { - return !strings.HasPrefix(urlPath, "/-/") && !strings.Contains(strings.TrimPrefix(urlPath, "/"), "/-/") -} - -// handleSecurity relays the npm audit endpoints to upstream. The request body -// is the dependency tree being audited, so no two requests share a cache key, -// and the response carries no tarball URLs to rewrite. -// -// Advisories come from upstream's database, not the proxy's own vulnerability -// data, and versions withheld by cooldown are not excluded from the report. -func (h *NPMHandler) handleSecurity(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - JSONError(w, http.StatusMethodNotAllowed, "method not allowed") - return - } - - if containsPathTraversal(r.URL.Path) { - JSONError(w, http.StatusBadRequest, "invalid path") - return - } - - body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, npmSecurityMaxBody)) - if err != nil { - var maxBytesErr *http.MaxBytesError - if errors.As(err, &maxBytesErr) { - JSONError(w, http.StatusRequestEntityTooLarge, "audit request too large") - return - } - // A client aborting mid-upload must not be told its payload was too big. - h.proxy.Logger.Warn("npm audit request body unreadable", "path", r.URL.Path, "error", err) - JSONError(w, http.StatusBadRequest, "could not read audit request") - return - } - - // EscapedPath keeps an encoded "?" or "#" from turning the rest of the path - // into a query or fragment upstream. - upstreamURL := h.upstreamURL + r.URL.EscapedPath() - if r.URL.RawQuery != "" { - upstreamURL += "?" + r.URL.RawQuery - } - - h.proxy.Logger.Info("npm audit request", "path", r.URL.Path, "bytes", len(body)) - - req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, upstreamURL, bytes.NewReader(body)) - if err != nil { - JSONError(w, http.StatusInternalServerError, "failed to create request") - return - } - - // Clients may gzip the body, so the headers describing it travel with it. - for _, header := range []string{headerContentType, headerContentEncoding, headerAccept, headerAcceptEncoding} { - if v := r.Header.Get(header); v != "" { - req.Header.Set(header, v) - } - } - if req.Header.Get(headerContentType) == "" { - req.Header.Set(headerContentType, contentTypeJSON) - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("npm audit request failed", "path", r.URL.Path, "error", err) - JSONError(w, http.StatusBadGateway, "failed to reach upstream registry") - return - } - defer func() { _ = resp.Body.Close() }() - - h.proxy.relayResponse(w, r, resp, nil) -} - -// sourceFor returns the registry serving a package and the key its metadata is -// cached under. -func (h *NPMHandler) sourceFor(packageName string) (registryURL, cacheKey string) { - if route, routed := h.routes.match(packageName); routed { - return route.url, route.cacheKey(packageName) - } - return h.upstreamURL, packageName -} - // handlePackageMetadata proxies package metadata from upstream and rewrites tarball URLs. func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { - packagePath := h.extractPackageName(r) - if packagePath == "" || containsPathTraversal(packagePath) { - JSONError(w, http.StatusBadRequest, "invalid package name") - return - } - packageName, registryPath := npmMetadataPath(packagePath) + packageName := h.extractPackageName(r) if packageName == "" { JSONError(w, http.StatusBadRequest, "invalid package name") return } - h.proxy.Logger.Debug("npm metadata request", "package", packageName, "path", packagePath) + h.proxy.Logger.Info("npm metadata request", "package", packageName) - registryURL, cacheKey := h.sourceFor(packageName) - // A version or dist-tag is its own path segment. Escaping the whole path - // turns "lodash/4.17.21" into "lodash%2F4.17.21", which the registry - // rejects with 405. - upstreamURL := registryURL + "/" + registryPath - metadataCacheKey := npmMetadataCacheKey(cacheKey, packagePath, packageName) + upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName)) - accept := h.packumentAccept() - - specifier, versionLookup := npmVersionSpecifier(packagePath, packageName) - if h.serveLocalMetadata(w, r, npmMetadataRequest{packageName, specifier, versionLookup, upstreamURL, accept, metadataCacheKey}) { - return + // Use abbreviated metadata when cooldown is disabled — it's much smaller + // (e.g. drizzle-orm: 4MB vs 92MB) but lacks the time map needed for cooldown. + accept := npmAbbreviatedCT + if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() { + accept = contentTypeJSON } - if stored, ok := h.proxy.storedRewrite("npm", metadataCacheKey, h.proxyURL, metadataCacheKey); ok { - serveRewrittenMetadata(w, r, stored) - return - } - - if versionLookup && h.npmVersionMetadataUsesFilteredPackument(packageName) { - h.handleFilteredNPMVersionMetadata(w, r, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey) - return - } - - if !versionLookup && h.serveStreamedPackument(w, r, packageName, upstreamURL, accept, metadataCacheKey) { - return - } - - var validate func([]byte) error - if !versionLookup { - validate = h.packumentValidator() - } - upstream := h.proxy.fetchMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept, validate) - if err := upstream.err; err != nil { + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, accept) + if err != nil { if errors.Is(err, ErrUpstreamNotFound) { - if versionLookup { - JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) - } else { - JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) - } - return - } - if writeUpstreamHeld(w, err) { + JSONError(w, http.StatusNotFound, "package not found") return } h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) @@ -293,463 +87,161 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, upstream.body, upstream.digest, func(b []byte) ([]byte, []string, error) { - return h.rewriteMetadataKeeping(packageName, b) - }) + rewritten, err := h.rewriteMetadata(packageName, body) if err != nil { - if r.Context().Err() != nil { - return // the client left while waiting on a shared rewrite - } - if errors.Is(err, ErrVersionDenied) { - JSONError(w, http.StatusForbidden, err.Error()) - return - } - if errors.Is(err, ErrVersionNotFound) { - JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) - return - } - if h.proxy.mustFilterMetadata("npm", packageName) { - h.proxy.Logger.Error("failed to filter npm metadata", "package", packageName, "error", err) - JSONError(w, http.StatusBadGateway, "failed to filter package metadata") - return - } // If rewriting fails, just proxy the original h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) - w.Header().Set(headerContentType, contentTypeJSON) + w.Header().Set("Content-Type", contentTypeJSON) w.WriteHeader(http.StatusOK) - _, _ = w.Write(upstream.body) + _, _ = w.Write(body) return } - serveRewrittenMetadata(w, r, rewritten) -} - -// serveStreamedPackument serves a packument rewritten from storage: the -// document is streamed from upstream into storage, then read twice, once to -// collect the time map and version names cooldown and the denylist decide -// on, and once to copy each kept version. registry.npmjs.org writes -// dist-tags before versions and time after them, so one pass can't do it. -// Neither the packument nor its rewrite is held in memory uncompressed, and -// packuments larger than metadata_max_size can be served. -// -// It reports false when it leaves the request to the in-memory rewrite: -// without metadata or rewrite caching, for a document it can't stream, such -// as an abbreviated packument where the full one is required, and when the -// rewrite fails. The document is stored by then, so the in-memory path reads -// it from storage. -func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, accept, metadataCacheKey string) bool { - if h.proxy.rewrites == nil { - return false - } - fullRequired := h.packumentValidator() != nil - options := h.npmRewriteOptions(packageName, nil) - var rewritten rewrittenMetadata - err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, func(entry *database.MetadataCacheEntry) (err error) { - rewritten, err = h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { - var scan *npm.Scan - err := readStored(open, func(in io.Reader) (err error) { - scan, err = npm.ScanPackument(in) - return err - }) - if err != nil { - return err - } - if fullRequired && scan.Abbreviated() { - return npm.ErrNotStreamable - } - return readStored(open, func(in io.Reader) error { - _, err := npm.RewriteStream(packageName, scan, in, out, options) - return err - }) - }) - return err - }) - switch { - case errors.Is(fetchErr, errInMemory): - return false - case fetchErr != nil: - h.writeFetchError(w, fetchErr) - return true - } - if err != nil { - if r.Context().Err() != nil { - return true // the client left while waiting on a shared rewrite - } - if !errors.Is(err, npm.ErrNotStreamable) { - h.proxy.Logger.Warn("failed to rewrite stored npm metadata, rewriting in memory", - "package", packageName, "error", err) - } - return false - } - serveRewrittenMetadata(w, r, rewritten) - return true -} - -// npmFetchError is a failure to fetch a packument, answered as such rather -// than as a failure to filter it. -type npmFetchError struct{ err error } - -func (e *npmFetchError) Error() string { return e.err.Error() } -func (e *npmFetchError) Unwrap() error { return e.err } - -// writeFetchError answers a request whose packument could not be fetched. -func (h *NPMHandler) writeFetchError(w http.ResponseWriter, err error) { - switch { - case errors.Is(err, ErrUpstreamNotFound): - JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) - case writeUpstreamHeld(w, err): - default: - h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) - JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") - } -} - -// errInMemory is returned when a packument has to be read into memory to -// answer the request: metadata is not cached in storage, or the stored -// document can't be read as a stream. -var errInMemory = errors.New("npm packument needs the in-memory path") - -// storedPackument returns the lookup of the packument stored for cacheKey, -// fetching it into storage first when needed. Its err is errInMemory when -// metadata isn't cached in storage, and ErrUpstreamNotFound and upstream -// errors as they come. -// With refresh, a stored copy within its TTL is fetched again rather than -// served. -func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool) metadataResult { - var gone string - if refresh && h.proxy.DB != nil { - if row, _ := h.proxy.DB.GetMetadataCache("npm", cacheKey); usableStoredMetadata(row) { - gone = row.ContentDigest.String - } - } - res := h.proxy.storedMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), gone) - if errors.Is(res.err, errMetadataNotStored) { - res.err = errInMemory - } - return res -} - -// readStoredPackument reads the packument stored for cacheKey with read, -// fetching it into storage first when needed. When the stored copy is gone -// or no longer has the digest its row records, the packument is fetched -// again and read once more. fetchErr is what storedPackument returned when the packument -// couldn't be fetched, and err is what read returned. -func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { - fetched, err := h.lookupStoredPackument(ctx, packumentURL, accept, cacheKey, false, read) - return err, fetched.err -} - -// lookupStoredPackument implements readStoredPackument. fetched is the -// lookup of the packument read, whose err is set when it couldn't be -// fetched. -func (h *NPMHandler) lookupStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool, read func(*database.MetadataCacheEntry) error) (fetched metadataResult, err error) { - fetched = h.storedPackument(ctx, packumentURL, accept, cacheKey, refresh) - if fetched.err != nil { - return fetched, nil - } - if err = read(fetched.entry); !errors.Is(err, storage.ErrNotFound) && !errors.Is(err, errStoredMetadataChanged) { - return fetched, err - } - h.proxy.Logger.Warn("stored npm metadata is missing or changed, fetching it again", "url", packumentURL, "error", err) - fetched = h.proxy.storedMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), fetched.entry.ContentDigest.String) - if errors.Is(fetched.err, errMetadataNotStored) { - fetched.err = errInMemory - } - if fetched.err != nil { - return fetched, nil - } - return fetched, read(fetched.entry) -} - -// packumentStreamValidator is packumentValidator for a packument read from -// storage. -func (h *NPMHandler) packumentStreamValidator() func(io.Reader) error { - if h.packumentValidator() == nil { - return nil - } - return func(r io.Reader) error { - entry, err := npm.LookupVersion(r, "") - if err != nil { - return err - } - if entry.Abbreviated { - return errAbbreviatedPackument - } - return nil - } -} - -// streamedVersionMetadata resolves a version or dist-tag through the -// packument stored in storage, as rewriteNPMVersionFromPackument does in -// memory: one pass collects what cooldown and the denylist decide on, and a -// second finds the version entry. It returns errInMemory when the request -// has to take the in-memory path. -func (h *NPMHandler) streamedVersionMetadata(ctx context.Context, packageName, specifier, packumentURL, accept, cacheKey, metadataCacheKey string) (rewrittenMetadata, error) { - if h.proxy.rewrites == nil { - return rewrittenMetadata{}, errInMemory - } - fullRequired := h.packumentValidator() != nil - options := h.npmRewriteOptions(packageName, nil) - var m rewrittenMetadata - err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) (err error) { - m, err = h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { - var scan *npm.Scan - err := readStored(open, func(in io.Reader) (err error) { - scan, err = npm.ScanPackument(in) - return err - }) - if err != nil { - return err - } - if fullRequired && scan.Abbreviated() { - return npm.ErrNotStreamable - } - return readStored(open, func(in io.Reader) error { - version, err := npm.VersionStream(packageName, specifier, scan, in, options) - if err != nil { - return npmRewriteError(err) - } - _, err = out.Write(version) - return err - }) - }) - return err - }) - switch { - case errors.Is(fetchErr, errInMemory): - return rewrittenMetadata{}, fetchErr - case fetchErr != nil: - return rewrittenMetadata{}, &npmFetchError{fetchErr} - } - switch { - case err == nil, ctx.Err() != nil, errors.Is(err, ErrVersionDenied), errors.Is(err, ErrVersionNotFound): - return m, err - case !errors.Is(err, npm.ErrNotStreamable): - h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", - "package", packageName, "error", err) - } - return rewrittenMetadata{}, errInMemory -} - -// readStored opens a stored document and reads it with read. -func readStored(open func() (io.ReadCloser, error), read func(io.Reader) error) error { - in, err := open() - if err != nil { - return err - } - defer func() { _ = in.Close() }() - return read(in) -} - -// packumentAccept returns the Accept header packuments are requested with. -// It prefers the smaller abbreviated packument format but includes -// application/json as a fallback so upstreams that reject the abbreviated type -// (e.g. JFrog Artifactory, which returns 406) can still respond with full -// metadata. When cooldown is enabled we must use full metadata exclusively -// because the abbreviated format omits the "time" map required for version age -// filtering. Operators can also force full metadata so clients that gate on -// publish age (for example Yarn's npmMinimalAgeGate) keep working through the -// proxy. -// -// Every request for a packument uses it, since they share one cache row: a -// request with another Accept would replace the document the others are -// served with one in the other format. -func (h *NPMHandler) packumentAccept() string { - if h.proxy.NPMFullMetadata || h.proxy.cooldownEnabled() { - return contentTypeJSON - } - return npmAcceptDefault -} - -// errAbbreviatedPackument rejects an abbreviated packument where the full one -// is needed. -var errAbbreviatedPackument = errors.New("abbreviated packument where full metadata is required") - -// packumentValidator returns the check a packument must pass to be served or -// cached, or nil when any document will do. Packuments share one cache row -// whatever format they were fetched in, so one cached in abbreviated form -// before cooldown or full metadata was turned on would otherwise still be -// served, and used for the cooldown tarball check, while it stayed fresh or -// as a stale copy when upstream failed. It has no "time" map, so cooldown -// would let every version through. Rejecting it makes the lookup fetch the -// full document instead. -func (h *NPMHandler) packumentValidator() func([]byte) error { - if h.packumentAccept() != contentTypeJSON { - return nil - } - return requireFullPackument -} - -// requireFullPackument rejects an abbreviated packument (see -// npm.Abbreviated). A full document without a time map, as some registries -// serve, is let through as before, and so is anything that is not a JSON -// object, leaving it to the usual handling. -func requireFullPackument(body []byte) error { - if npm.Abbreviated(body) { - return errAbbreviatedPackument - } - return nil -} - -// handleFilteredNPMVersionMetadata serves /pkg/ and /pkg/ from the -// filtered packument so cooldown and dist-tags match the root metadata response. -func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey string) { - _, packumentPath := npmMetadataPath(packageName) - packumentURL := registryURL + "/" + packumentPath - rewritten, err := h.streamedVersionMetadata(r.Context(), packageName, specifier, packumentURL, accept, cacheKey, metadataCacheKey) - var fetchErr *npmFetchError - switch { - case errors.As(err, &fetchErr): - h.writeFetchError(w, fetchErr.err) - return - case !errors.Is(err, errInMemory): - h.writeVersionMetadata(w, r, packageName, rewritten, err) - return - } - - packument, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) - if err != nil { - if errors.Is(err, ErrUpstreamNotFound) { - JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) - JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") - return - } - - rewritten, err = h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { - out, err := h.rewriteNPMVersionFromPackument(packageName, specifier, b) - return out, nil, err - }) - h.writeVersionMetadata(w, r, packageName, rewritten, err) -} - -// writeVersionMetadata answers a version metadata request with its rewrite -// or the error that kept it from being made. -func (h *NPMHandler) writeVersionMetadata(w http.ResponseWriter, r *http.Request, packageName string, rewritten rewrittenMetadata, err error) { - if err != nil { - if r.Context().Err() != nil { - return - } - if errors.Is(err, ErrVersionDenied) { - JSONError(w, http.StatusForbidden, err.Error()) - return - } - if errors.Is(err, ErrVersionNotFound) { - JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) - return - } - if h.proxy.mustFilterMetadata("npm", packageName) { - h.proxy.Logger.Error("failed to filter npm metadata", "package", packageName, "error", err) - JSONError(w, http.StatusBadGateway, "failed to filter package metadata") - return - } - h.proxy.Logger.Warn("failed to rewrite npm version metadata", "error", err) - JSONError(w, http.StatusBadGateway, "failed to filter package metadata") - return - } - - w.Header().Set(headerContentType, contentTypeJSON) + w.Header().Set("Content-Type", contentTypeJSON) w.WriteHeader(http.StatusOK) - _, _ = w.Write(rewritten.content()) + _, _ = w.Write(rewritten) } -// npmMetadataCacheKey qualifies routed-package metadata cache entries with the -// requested version or dist-tag when the client asks for a single version. -func npmMetadataCacheKey(cacheKey, packagePath, packageName string) string { - specifier, versionLookup := npmVersionSpecifier(packagePath, packageName) - if !versionLookup { - return cacheKey +// rewriteMetadata rewrites tarball URLs in npm package metadata to point at this proxy. +// If cooldown is enabled, versions published too recently are filtered out. +func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, error) { + var metadata map[string]any + if err := json.Unmarshal(body, &metadata); err != nil { + return nil, err } - return cacheKey + "/" + specifier -} -// npmVersionSpecifier returns the version or dist-tag segment after the package -// name in a metadata path, for example "4.17.21" or "latest". -func npmVersionSpecifier(packagePath, packageName string) (string, bool) { - if packagePath == packageName { - return "", false + // Rewrite tarball URLs in versions + versions, ok := metadata["versions"].(map[string]any) + if !ok { + return body, nil // No versions to rewrite } - specifier, ok := strings.CutPrefix(packagePath, packageName+"/") - return specifier, ok + + h.applyCooldownFiltering(metadata, versions, packageName) + h.rewriteTarballURLs(versions, packageName) + + return json.Marshal(metadata) } -func (h *NPMHandler) npmVersionMetadataUsesFilteredPackument(packageName string) bool { - if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() { - return true +// applyCooldownFiltering removes versions that are too recently published, +// and updates dist-tags.latest if the current latest was filtered out. +func (h *NPMHandler) applyCooldownFiltering(metadata map[string]any, versions map[string]any, packageName string) { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return } - if h.proxy.Denylist != nil { - return len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 + + timeMap, _ := metadata["time"].(map[string]any) + if timeMap == nil { + return } - return false + + packagePURL := purl.MakePURLString("npm", packageName, "") + + for version := range versions { + publishedStr, ok := timeMap[version].(string) + if !ok { + continue + } + publishedAt, err := time.Parse(time.RFC3339, publishedStr) + if err != nil { + continue + } + if !h.proxy.Cooldown.IsAllowed("npm", packagePURL, publishedAt) { + h.proxy.Logger.Info("cooldown: filtering npm version", + "package", packageName, "version", version, + "published", publishedStr) + delete(versions, version) + delete(timeMap, version) + } + } + + h.updateDistTagsLatest(metadata, versions, timeMap) } -// rewriteNPMVersionFromPackument picks one version entry out of an already -// filtered packument and rewrites its tarball URL. -func (h *NPMHandler) rewriteNPMVersionFromPackument(packageName, specifier string, packument []byte) ([]byte, error) { - out, err := npm.Version(packageName, specifier, packument, h.npmRewriteOptions(packageName, nil)) - return out, npmRewriteError(err) -} +// updateDistTagsLatest updates the dist-tags.latest field if the current latest +// version was removed by cooldown filtering. +func (h *NPMHandler) updateDistTagsLatest(metadata, versions, timeMap map[string]any) { + distTags, ok := metadata["dist-tags"].(map[string]any) + if !ok { + return + } -// rewriteMetadataKeeping rewrites tarball URLs in npm package metadata to -// point at this proxy, filters the versions cooldown and the denylist -// withhold, and reports the versions it kept. See npm.Rewrite. -func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([]byte, []string, error) { - return h.rewriteMetadataWithLocal(packageName, body, nil) -} + latest, ok := distTags["latest"].(string) + if !ok { + return + } -// rewriteMetadataWithLocal is rewriteMetadataKeeping that also lists the -// pushed versions in local, in place of any upstream entry with the same -// version. Cooldown and the denylist only filter upstream versions, and -// dist-tags stay as upstream has them. -func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, local []npmLocalEntry) ([]byte, []string, error) { - out, kept, err := npm.Rewrite(packageName, body, h.npmRewriteOptions(packageName, local)) - return out, kept, npmRewriteError(err) -} + if _, exists := versions[latest]; exists { + return + } -// npmRewriteOptions gathers what a rewrite of packageName's metadata depends -// on from the proxy: its URL, the cooldown and denylist policy, the time -// cooldown is evaluated at, and the pushed versions to list. The callbacks -// log what the rewrite did. -func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry) npm.Options { - return npm.Options{ - ProxyURL: h.proxyURL, - Cooldown: h.proxy.cooldown(), - DeniedVersions: h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName)), - EvaluatedAt: time.Now(), - LocalVersions: local, - // Logged once per document: a line per version flooded debug logs, - // since packages such as eslint list hundreds of versions and with - // cooldown on every request rewrites the document again. - OnTarballsRewritten: func(count int) { - h.proxy.Logger.Debug("rewrote tarball URLs", "package", packageName, "count", count) - }, - OnCooldownFilter: func(version, publishedAt string) { - h.proxy.Logger.Info("cooldown: filtering npm version", "package", packageName, "version", version, - "published", publishedAt) - }, + if newLatest := h.findNewestVersion(versions, timeMap); newLatest != "" { + distTags["latest"] = newLatest } } -// npmRewriteError maps the rewrite's errors to the handler's, which the -// routes answer with 403 and 404. -func npmRewriteError(err error) error { - switch { - case errors.Is(err, npm.ErrVersionDenied): - return ErrVersionDenied - case errors.Is(err, npm.ErrVersionNotFound): - return ErrVersionNotFound - default: - return err +// rewriteTarballURLs rewrites all tarball URLs in version entries to point at this proxy. +func (h *NPMHandler) rewriteTarballURLs(versions map[string]any, packageName string) { + for version, vdata := range versions { + vmap, ok := vdata.(map[string]any) + if !ok { + continue + } + + dist, ok := vmap["dist"].(map[string]any) + if !ok { + continue + } + + tarball, ok := dist["tarball"].(string) + if !ok { + continue + } + + filename := tarball + if idx := strings.LastIndex(tarball, "/"); idx >= 0 { + filename = tarball[idx+1:] + } + + escapedName := url.PathEscape(packageName) + newTarball := fmt.Sprintf("%s/npm/%s/-/%s", h.proxyURL, escapedName, filename) + dist["tarball"] = newTarball + + h.proxy.Logger.Debug("rewrote tarball URL", + "package", packageName, "version", version, + "old", tarball, "new", newTarball) } } -func (h *NPMHandler) proxyTarballURL(packageName, version, tarball string) string { - return npm.TarballURL(h.proxyURL, packageName, version, tarball) +// findNewestVersion returns the version string with the most recent timestamp +// from the remaining versions, using the time map. +func (h *NPMHandler) findNewestVersion(versions map[string]any, timeMap map[string]any) string { + if timeMap == nil { + return "" + } + + type versionTime struct { + version string + t time.Time + } + + var vts []versionTime + for v := range versions { + if ts, ok := timeMap[v].(string); ok { + if t, err := time.Parse(time.RFC3339, ts); err == nil { + vts = append(vts, versionTime{v, t}) + } + } + } + + if len(vts) == 0 { + return "" + } + + sort.Slice(vts, func(i, j int) bool { + return vts[i].t.After(vts[j].t) + }) + + return vts[0].version } // handleDownload serves a package tarball, fetching and caching from upstream if needed. @@ -768,325 +260,17 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - h.proxy.Logger.Debug("npm download request", + h.proxy.Logger.Info("npm download request", "package", packageName, "version", version, "filename", filename) - if h.proxy.versionDenied("npm", packageName, version) { - JSONError(w, http.StatusForbidden, ErrVersionDenied.Error()+": "+canonicalVersionPURL("npm", packageName, version)) - return - } - // Pushed versions are listed without cooldown, so they are served without - // it too, even when the upstream published the same version recently. - local, err := h.proxy.GetLocalArtifact(r.Context(), "npm", packageName, version, filename) + result, err := h.proxy.GetOrFetchArtifact(r.Context(), "npm", packageName, version, filename) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to read pushed package") - return - } - if local != nil { - ServeArtifactRequest(w, r, local) - return - } - if h.proxy.hostedOnly("npm", packageName) { - JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) - return - } - registryURL, cacheKey := h.sourceFor(packageName) - metadata := sync.OnceValues(func() (npmVersionData, error) { - return h.versionData(r.Context(), packageName, registryURL+"/"+url.PathEscape(packageName), cacheKey, version) - }) - inCooldown, err := h.versionInCooldown(packageName, version, metadata) - if err != nil { - if h.proxy.serveCachedDespiteCooldownError(w, r, "npm", packageName, version, h.routes.cacheFilename(packageName, filename), err) { - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Warn("cooldown: could not read npm publish time", - "package", packageName, "version", version, "error", err) - JSONError(w, http.StatusBadGateway, "failed to check cooldown") - return - } - if inCooldown { - h.proxy.Logger.Info("cooldown: withholding npm tarball", - "package", packageName, "version", version) - JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) + h.proxy.Logger.Error("failed to get artifact", "error", err) + JSONError(w, http.StatusBadGateway, "failed to fetch package") return } - result, err := h.getTarball(r, registryURL, packageName, version, filename, metadata) - if err != nil { - switch { - case errors.Is(err, ErrUpstreamNotFound): - JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) - case errors.Is(err, ErrArtifactBlocked), errors.Is(err, ErrVersionDenied): - JSONError(w, http.StatusForbidden, err.Error()) - default: - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("failed to get artifact", "error", err) - JSONError(w, http.StatusBadGateway, "failed to fetch package") - } - return - } - - ServeArtifactRequest(w, r, result) -} - -func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, version, filename string, metadata func() (npmVersionData, error)) (*CacheResult, error) { - cacheFilename := h.routes.cacheFilename(packageName, filename) - if cached, err := h.proxy.GetCachedArtifact(r.Context(), "npm", packageName, version, cacheFilename); err != nil || cached != nil { - return cached, err - } - // The packument says where the tarball is and what it must hash to. - // Without it the tarball could not be verified, and an unverified copy - // would be cached and served to everyone, so the download fails instead. - data, err := metadata() - if err != nil { - return nil, fmt.Errorf("fetching npm metadata to verify tarball: %w", err) - } - downloadURL := fmt.Sprintf("%s/%s/-/%s", registryURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) - if tarball := npmEntryTarball(data.entry); tarball != "" { - downloadURL, err = validateTarballURL(tarball, registryURL) - if err != nil { - return nil, err - } - } - sri := npmEntryIntegrity(data.entry) - return h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "npm", packageName, version, cacheFilename, downloadURL, sri) -} - -// npmEntryIntegrity returns the integrity a version's packument entry -// declares for its tarball: dist.integrity when it is a Subresource Integrity string -// the proxy can verify, or else the SHA-1 in dist.shasum, which is all -// versions published before integrity existed carry, as "sha1-". It -// returns "" when neither is usable, and the tarball is fetched unverified as -// before. -func npmEntryIntegrity(entry []byte) string { - sri, _, _ := jsonscan.LookupString(entry, "dist", "integrity") - if sri != "" { - if _, err := integrity.ParseSRI(sri); err == nil { - return sri - } - } - shasum, _, _ := jsonscan.LookupString(entry, "dist", "shasum") - return sha1Checksum(shasum) -} - -// npmVersionTarball returns versions[version].dist.tarball from a packument, -// or "" if it has none. It reads that one value in place rather than decoding -// every version, since it runs on each tarball the proxy has not cached yet. -func npmVersionTarball(body []byte, version string) string { - entry, _ := jsonscan.Lookup(body, "versions", version) - return npmEntryTarball(entry) -} - -// npmEntryTarball is npmVersionTarball for the version's entry. -func npmEntryTarball(entry []byte) string { - tarball, _, _ := jsonscan.LookupString(entry, "dist", "tarball") - return tarball -} - -// npmVersionData is what a tarball download reads from the packument: the -// version's entry, nil when it has none, and its publish time from the time -// map, "" when it has none. -type npmVersionData struct { - entry []byte - published string -} - -// versionData reads what a tarball download needs about version from the -// packument: from storage in one pass when metadata is cached there, and -// otherwise from the packument read into memory. -// -// A packument cached within its TTL can predate the version. Under cooldown a -// missing entry would let the version through with no publish time, so the -// packument is fetched again and the version decided on what upstream lists -// now, as the Composer handler does. -func (h *NPMHandler) versionData(ctx context.Context, packageName, packumentURL, cacheKey, version string) (npmVersionData, error) { - data, fromCache, err := h.readVersionData(ctx, packumentURL, cacheKey, version, false) - if err == nil && data.entry == nil && fromCache && h.proxy.cooldownApplies("npm", packageName) { - h.proxy.Logger.Info("cooldown: cached npm metadata lacks version, fetching it again", - "package", packageName, "version", version) - data, _, err = h.readVersionData(ctx, packumentURL, cacheKey, version, true) - } - return data, err -} - -// readVersionData implements versionData. fromCache reports whether the -// packument was a cached copy, within its TTL or served stale; refresh -// fetches it from upstream. -func (h *NPMHandler) readVersionData(ctx context.Context, packumentURL, cacheKey, version string, refresh bool) (data npmVersionData, fromCache bool, err error) { - accept := h.packumentAccept() - var found npm.VersionEntry - fetched, err := h.lookupStoredPackument(ctx, packumentURL, accept, cacheKey, refresh, func(entry *database.MetadataCacheEntry) error { - return readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { - found, err = npm.LookupVersion(in, version) - return err - }) - }) - fetchErr := fetched.err - switch { - case fetchErr == nil && err == nil && (!found.Abbreviated || h.packumentValidator() == nil): - if refresh && fetched.stale { - return npmVersionData{}, false, errRefetchFailed(fetched) - } - return npmVersionData{entry: found.Entry, published: found.Published}, fetched.fromCache || fetched.stale, nil - case fetchErr == nil && err != nil: - h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) - case fetchErr != nil && !errors.Is(fetchErr, errInMemory): - return npmVersionData{}, false, fetchErr - } - - var res metadataResult - if refresh { - res = h.proxy.refreshMetadata(ctx, "npm", cacheKey, packumentURL, accept, "", h.packumentValidator()) - } else { - res = h.proxy.fetchMetadataDocument(ctx, "npm", cacheKey, packumentURL, accept, h.packumentValidator()) - } - if res.err != nil { - return npmVersionData{}, false, res.err - } - if refresh && res.stale { - return npmVersionData{}, false, errRefetchFailed(res) - } - entry, _ := jsonscan.Lookup(res.body, "versions", version) - published, _, err := jsonscan.LookupString(res.body, "time", version) - if err != nil { - return npmVersionData{}, false, fmt.Errorf("parsing npm metadata: %w", err) - } - return npmVersionData{entry: entry, published: published}, res.fromCache || res.stale, nil -} - -// errRefetchFailed is the error for a refetch that got only the stale copy -// back. That copy is what lacked the version, so it can't decide it. -func errRefetchFailed(res metadataResult) error { - return fmt.Errorf("npm metadata could not be fetched again: %w", res.upstreamErr) -} - -func validateTarballURL(raw, registryURL string) (string, error) { - return validateRegistryURL("tarball", raw, registryURL) -} - -// validateRegistryURL checks that a URL a packument lists for a version's -// file is on the package's registry, under its base path, and returns it. -// what names the file in errors. -func validateRegistryURL(what, raw, registryURL string) (string, error) { - parsed, err := url.Parse(raw) - if err != nil { - return "", fmt.Errorf("parsing npm %s URL: %w", what, err) - } - upstream, err := url.Parse(registryURL) - if err != nil { - return "", fmt.Errorf("parsing npm upstream URL: %w", err) - } - if parsed.User != nil || parsed.Fragment != "" || - parsed.Scheme != upstream.Scheme || !strings.EqualFold(parsed.Host, upstream.Host) { - return "", fmt.Errorf("npm %s URL does not match upstream registry", what) - } - if containsPathTraversal(parsed.Path) || strings.Contains(parsed.Path, "\\") { - return "", fmt.Errorf("npm %s URL contains path traversal", what) - } - basePath := strings.TrimRight(upstream.Path, "/") - if basePath != "" && !strings.HasPrefix(parsed.Path, basePath+"/") { - return "", fmt.Errorf("npm %s URL is outside upstream base path", what) - } - return parsed.String(), nil -} - -// versionInCooldown reports whether a version is still inside the cooldown -// window. Filtering the packument is not enough on its own: tarball URLs are -// predictable and lockfiles record them directly, so `npm ci` reaches the -// download path without ever requesting metadata. -// -// A version's publish time is immutable, so the check reads the stored -// versions row first and only falls back to the packument for a version the -// proxy has never seen, persisting the parsed time so the packument is -// fetched and parsed at most once per version. A version with no usable -// publish time is allowed through, matching how applyCooldownFiltering -// treats it. A packument that can't be fetched or parsed is an error, so a -// failing upstream can't let a version through. -// -// The versions row is keyed by the plain version PURL, which does not say -// which registry the version came from, so routed packages leave it alone: -// their publish time always comes from the route's own metadata, cached under -// the route. Otherwise a time read from the default registry would be trusted -// for a private version of the same name, a private time would hold back the -// public version, and moving a route to another upstream would keep the old -// one's times. -func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() (npmVersionData, error)) (bool, error) { - if !h.proxy.cooldownApplies("npm", packageName) { - return false, nil - } - - _, routed := h.routes.match(packageName) - versionPURL := canonicalVersionPURL("npm", packageName, version) - if !routed { - if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return !h.proxy.cooldown().IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time), nil - } - } - - data, err := metadata() - if errors.Is(err, ErrUpstreamNotFound) { - return false, nil - } - if err != nil { - return false, err - } - - published := data.published - if published == "" { - return false, nil - } - - publishedAt, err := time.Parse(time.RFC3339, published) - if err != nil { - return false, nil - } - - if !routed { - if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, canonicalPackagePURL("npm", packageName), publishedAt); err != nil { - h.proxy.Logger.Warn("cooldown: could not store npm publish time", - "package", packageName, "version", version, "error", err) - } - } - - return !h.proxy.cooldown().IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt), nil -} - -// npmMetadataPath returns the package name and the encoded registry path. -// Examples: lodash → lodash; lodash/4.17.21 → lodash/4.17.21; -// @babel/core → @babel%2Fcore; @babel/core/7.23.0 → @babel%2Fcore/7.23.0. -func npmMetadataPath(packagePath string) (packageName, registryPath string) { - // GET /npm/ has no package segment; handlePackageMetadata rejects that earlier. - if packagePath == "" { - return "", "" - } - if !strings.Contains(packagePath, "/") { - return packagePath, url.PathEscape(packagePath) - } - if strings.HasPrefix(packagePath, "@") { - scope, rest, _ := strings.Cut(packagePath, "/") - name, version, ok := strings.Cut(rest, "/") - packageName = scope + "/" + name - registryPath = url.PathEscape(scope) + "%2F" + url.PathEscape(name) - if ok { - registryPath += "/" + url.PathEscape(version) - } - return packageName, registryPath - } - name, version, _ := strings.Cut(packagePath, "/") - return name, url.PathEscape(name) + "/" + url.PathEscape(version) -} - -func escapeNPMDownloadPackage(packageName string) string { - scope, name, scoped := strings.Cut(packageName, "/") - if scoped && strings.HasPrefix(scope, "@") && len(scope) > 1 && name != "" && !strings.Contains(name, "/") { - return url.PathEscape(scope) + "/" + url.PathEscape(name) - } - return url.PathEscape(packageName) + ServeArtifact(w, result) } // extractPackageName extracts the package name from the request path. @@ -1129,6 +313,28 @@ func (h *NPMHandler) parseDownloadPath(path string) (packageName, filename strin return packageName, filename } +// extractVersionFromFilename extracts version from npm tarball filename. +// e.g., "lodash-4.17.21.tgz" -> "4.17.21" +// e.g., "core-7.23.0.tgz" for @babel/core -> "7.23.0" func (h *NPMHandler) extractVersionFromFilename(packageName, filename string) string { - return npm.VersionFromFilename(packageName, filename) + // Remove .tgz extension + if !strings.HasSuffix(filename, ".tgz") { + return "" + } + base := strings.TrimSuffix(filename, ".tgz") + + // For scoped packages, the filename uses the short name + shortName := packageName + if strings.Contains(packageName, "/") { + parts := strings.SplitN(packageName, "/", scopedParts) + shortName = parts[1] + } + + // Expected format: {shortName}-{version} + prefix := shortName + "-" + if !strings.HasPrefix(base, prefix) { + return "" + } + + return strings.TrimPrefix(base, prefix) } diff --git a/internal/handler/npm_attestations.go b/internal/handler/npm_attestations.go deleted file mode 100644 index cc84003..0000000 --- a/internal/handler/npm_attestations.go +++ /dev/null @@ -1,110 +0,0 @@ -package handler - -import ( - "errors" - "net/http" - "net/url" - "strings" - "sync" - - "github.com/git-pkgs/proxy/internal/jsonscan" -) - -// errNoAttestations answers a request for a version whose packument entry -// lists no attestations. -var errNoAttestations = errors.New("attestations not found") - -// handleAttestations serves /-/npm/v1/attestations/{name}@{version}, which -// `npm audit signatures` reads to verify provenance. Packuments point -// dist.attestations.url here (see npm.AttestationsURL). -func (h *NPMHandler) handleAttestations(w http.ResponseWriter, r *http.Request) { - packageName, version, ok := parseNPMAttestationsPath(strings.TrimPrefix(r.URL.Path, npmAttestationsPrefix)) - if !ok { - JSONError(w, http.StatusBadRequest, "invalid package name") - return - } - - if h.proxy.versionDenied("npm", packageName, version) { - JSONError(w, http.StatusForbidden, ErrVersionDenied.Error()+": "+canonicalVersionPURL("npm", packageName, version)) - return - } - local, err := h.npmLocalEntries(packageName) - if err != nil { - h.proxy.Logger.Error("failed to read pushed npm versions", "package", packageName, "error", err) - JSONError(w, http.StatusInternalServerError, "failed to read pushed versions") - return - } - if _, pushed := findNPMLocalEntry(local, version); pushed || h.proxy.hostedOnly("npm", packageName) { - JSONError(w, http.StatusNotFound, errNoAttestations.Error()) - return - } - - registryURL, cacheKey := h.sourceFor(packageName) - metadata := sync.OnceValues(func() (npmVersionData, error) { - return h.versionData(r.Context(), packageName, registryURL+"/"+url.PathEscape(packageName), cacheKey, version) - }) - - inCooldown, err := h.versionInCooldown(packageName, version, metadata) - if err != nil { - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Warn("cooldown: could not read npm publish time", - "package", packageName, "version", version, "error", err) - JSONError(w, http.StatusBadGateway, "failed to check cooldown") - return - } - if inCooldown { - JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) - return - } - - data, err := metadata() - if err != nil { - if errors.Is(err, ErrUpstreamNotFound) { - JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) - JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") - return - } - listed, _, _ := jsonscan.LookupString(data.entry, "dist", "attestations", "url") - if listed == "" { - JSONError(w, http.StatusNotFound, errNoAttestations.Error()) - return - } - upstreamURL, err := validateRegistryURL("attestations", listed, registryURL) - if err != nil { - // Mirrors of npmjs can keep its URLs in packuments. The proxy only - // ever asks the package's own registry. - upstreamURL = registryURL + "/-/npm/v1/attestations/" + url.PathEscape(packageName) + "@" + url.PathEscape(version) - } - - h.proxy.proxyImmutable(w, r, upstreamURL, "npm", cacheKey+"/-/attestations/"+version, contentTypeJSON) -} - -// parseNPMAttestationsPath splits "{name}@{version}" as the attestations -// endpoint takes it, where name may be "@scope/name" with the slash encoded -// or not. It rejects anything that is not one package and one version. -func parseNPMAttestationsPath(spec string) (packageName, version string, ok bool) { - at := strings.LastIndex(spec, "@") - if at <= 0 { - return "", "", false - } - packageName, version = spec[:at], spec[at+1:] - if version == "" || strings.ContainsAny(version, "/\\") || containsPathTraversal(packageName) || containsPathTraversal(version) { - return "", "", false - } - scope, name, scoped := strings.Cut(packageName, "/") - switch { - case scoped: - ok = len(scope) > 1 && scope[0] == '@' && name != "" && !strings.ContainsAny(name, "/@") - default: - ok = !strings.ContainsAny(packageName, "@\\") - } - return packageName, version, ok -} diff --git a/internal/handler/npm_attestations_test.go b/internal/handler/npm_attestations_test.go deleted file mode 100644 index 3717eaf..0000000 --- a/internal/handler/npm_attestations_test.go +++ /dev/null @@ -1,191 +0,0 @@ -package handler - -import ( - "net/http" - "slices" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -const testAttestations = `{"attestations":[{"predicateType":"https://slsa.dev/provenance/v1","bundle":{}}]}` - -// attestedPackument returns a packument whose version lists attestations at -// attestationsURL, resolved against the upstream's URL when it starts with -// "/", and was published age ago. -func attestedPackument(name, version, attestationsURL string, age time.Duration) func(string) string { - published := time.Now().Add(-age).UTC().Format(time.RFC3339) - return func(baseURL string) string { - if strings.HasPrefix(attestationsURL, "/") { - attestationsURL = baseURL + attestationsURL - } - dist := `{"tarball":"` + baseURL + "/" + name + `/-/x-` + version + `.tgz"` - if attestationsURL != "" { - dist += `,"attestations":{"url":"` + attestationsURL + `","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}}` - } - dist += "}" - return `{"name":"` + name + `","dist-tags":{"latest":"` + version + `"},` + - `"versions":{"` + version + `":{"version":"` + version + `","dist":` + dist + `}},` + - `"time":{"` + version + `":"` + published + `"}}` - } -} - -func attestationsFile(string) string { return testAttestations } - -func newAttestationsProxy(t *testing.T) *Proxy { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - return proxy -} - -func TestNPMAttestationsServedAndCached(t *testing.T) { - upstream := newNPMUpstream(t, map[string]func(string) string{ - "/sigstore": attestedPackument("sigstore", "3.0.0", "/-/npm/v1/attestations/sigstore@3.0.0", 30*24*time.Hour), - "/-/npm/v1/attestations/sigstore@3.0.0": attestationsFile, - }, http.StatusOK) - h := NewNPMHandler(newAttestationsProxy(t), "http://proxy.test", upstream.URL) - - for range 2 { - w := serveNPM(t, h, http.MethodGet, "/-/npm/v1/attestations/sigstore@3.0.0", nil, nil) - if w.Code != http.StatusOK || w.Body.String() != testAttestations { - t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) - } - } - fetched := 0 - for _, p := range upstream.requested() { - if p == "/-/npm/v1/attestations/sigstore@3.0.0" { - fetched++ - } - } - if fetched != 1 { - t.Errorf("attestations fetched %d times, want once: %v", fetched, upstream.requested()) - } -} - -func TestNPMAttestationsRoutedPackageStaysOnRoute(t *testing.T) { - public := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) - private := newNPMUpstream(t, map[string]func(string) string{ - routedNPMPath: attestedPackument("@example/widgets", "1.0.0", "/registry/-/npm/v1/attestations/@example%2fwidgets@1.0.0", 0), - "/registry/-/npm/v1/attestations/@example/widgets@1.0.0": attestationsFile, - }, http.StatusOK) - h := newRoutedNPMHandler(newAttestationsProxy(t), public, private) - - for _, target := range []string{ - "/-/npm/v1/attestations/@example%2fwidgets@1.0.0", - "/-/npm/v1/attestations/@example/widgets@1.0.0", - } { - w := serveNPM(t, h, http.MethodGet, target, nil, nil) - if w.Code != http.StatusOK || w.Body.String() != testAttestations { - t.Errorf("%s: status = %d, body = %s", target, w.Code, w.Body.String()) - } - } - if got := public.requested(); len(got) != 0 { - t.Errorf("default registry was queried for a routed package: %v", got) - } -} - -// TestNPMAttestationsOnlyAskThePackageRegistry checks that an attestations -// URL on another host is not followed: the proxy asks the package's registry -// at its standard path instead. -func TestNPMAttestationsOnlyAskThePackageRegistry(t *testing.T) { - other := newNPMUpstream(t, map[string]func(string) string{"/-/npm/v1/attestations/demo@1.0.0": attestationsFile}, http.StatusOK) - upstream := newNPMUpstream(t, map[string]func(string) string{ - "/demo": attestedPackument("demo", "1.0.0", other.URL+"/-/npm/v1/attestations/demo@1.0.0", 0), - "/-/npm/v1/attestations/demo@1.0.0": attestationsFile, - }, http.StatusOK) - h := NewNPMHandler(newAttestationsProxy(t), "http://proxy.test", upstream.URL) - - w := serveNPM(t, h, http.MethodGet, "/-/npm/v1/attestations/demo@1.0.0", nil, nil) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) - } - if got := other.requested(); len(got) != 0 { - t.Errorf("host named in the packument was queried: %v", got) - } - if !slices.Contains(upstream.requested(), "/-/npm/v1/attestations/demo@1.0.0") { - t.Errorf("registry not asked for attestations: %v", upstream.requested()) - } -} - -func TestNPMAttestationsWithheld(t *testing.T) { - tests := []struct { - name string - packument func(string) string - setup func(t *testing.T, p *Proxy) - wantStatus int - }{ - {"version has none", attestedPackument("demo", "1.0.0", "", 0), nil, http.StatusNotFound}, - {"version in cooldown", attestedPackument("demo", "1.0.0", "/-/npm/v1/attestations/demo@1.0.0", time.Hour), - func(_ *testing.T, p *Proxy) { p.Cooldown = &cooldown.Config{Default: "7d"} }, http.StatusNotFound}, - {"version denied", attestedPackument("demo", "1.0.0", "/-/npm/v1/attestations/demo@1.0.0", 0), - func(t *testing.T, p *Proxy) { setTestDenylist(t, p, "pkg:npm/demo@1.0.0") }, http.StatusForbidden}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - upstream := newNPMUpstream(t, map[string]func(string) string{ - "/demo": tt.packument, - "/-/npm/v1/attestations/demo@1.0.0": attestationsFile, - }, http.StatusOK) - proxy := newAttestationsProxy(t) - if tt.setup != nil { - tt.setup(t, proxy) - } - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - - w := serveNPM(t, h, http.MethodGet, "/-/npm/v1/attestations/demo@1.0.0", nil, nil) - if w.Code != tt.wantStatus { - t.Errorf("status = %d, want %d; body: %s", w.Code, tt.wantStatus, w.Body.String()) - } - if slices.Contains(upstream.requested(), "/-/npm/v1/attestations/demo@1.0.0") { - t.Error("attestations were fetched for a withheld version") - } - }) - } -} - -func TestNPMAttestationsRejectsInvalidPaths(t *testing.T) { - upstream := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) - h := NewNPMHandler(newAttestationsProxy(t), "http://proxy.test", upstream.URL) - - for _, target := range []string{ - "/-/npm/v1/attestations/demo", - "/-/npm/v1/attestations/demo@", - "/-/npm/v1/attestations/@demo@1.0.0", - "/-/npm/v1/attestations/@scope/a/b@1.0.0", - "/-/npm/v1/attestations/demo@1.0.0/x", - "/-/npm/v1/attestations/..%2f..%2fetc@1.0.0", - } { - w := serveNPM(t, h, http.MethodGet, target, nil, nil) - if w.Code != http.StatusBadRequest { - t.Errorf("%s: status = %d, want 400", target, w.Code) - } - } - if got := upstream.requested(); len(got) != 0 { - t.Errorf("upstream queried for invalid paths: %v", got) - } -} - -func TestNPMMetadataRewritesAttestationsURL(t *testing.T) { - const attestations = "/registry/-/npm/v1/attestations/@example%2fwidgets@1.0.0" - upstream := newNPMUpstream(t, map[string]func(string) string{ - routedNPMPath: attestedPackument("@example/widgets", "1.0.0", attestations, 0), - routedNPMPath + "/1.0.0": func(baseURL string) string { - return `{"name":"@example/widgets","version":"1.0.0","dist":{"attestations":{"url":"` + baseURL + attestations + - `"},"tarball":"` + baseURL + `/registry/@example/widgets/-/widgets-1.0.0.tgz"}}` - }, - }, http.StatusOK) - h := newRoutedNPMHandler(newAttestationsProxy(t), upstream, upstream) - - for _, target := range []string{"/@example%2fwidgets", "/@example%2fwidgets/1.0.0"} { - w := serveNPM(t, h, http.MethodGet, target, nil, nil) - want := `"url":"http://proxy.test/npm/-/npm/v1/attestations/@example%2Fwidgets@1.0.0"` - if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), want) { - t.Errorf("%s: status %d, attestations URL not rewritten: %s", target, w.Code, w.Body.String()) - } - } -} diff --git a/internal/handler/npm_dist_tags.go b/internal/handler/npm_dist_tags.go deleted file mode 100644 index 210b45f..0000000 --- a/internal/handler/npm_dist_tags.go +++ /dev/null @@ -1,115 +0,0 @@ -package handler - -import ( - "context" - "errors" - "net/http" - "strings" - - "github.com/git-pkgs/proxy/internal/jsonscan" -) - -// npmPackagePrefix and npmDistTagsSuffix frame the path `npm dist-tag ls` -// reads: /-/package/{name}/dist-tags, with a scoped name's slash encoded. -const ( - npmPackagePrefix = "/-/package/" - npmDistTagsSuffix = "/dist-tags" -) - -// handleDistTags answers `npm dist-tag ls` from the dist-tags of the -// packument the proxy serves, not from upstream's, so a tag never names a -// version cooldown or the denylist holds back, and pushed versions and -// package routes count as they do for the packument. Changing tags is not -// supported; Routes answers anything but GET with 405. -func (h *NPMHandler) handleDistTags(w http.ResponseWriter, r *http.Request) { - packageName := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, npmPackagePrefix), npmDistTagsSuffix) - if !validNPMPackageName(packageName) { - JSONError(w, http.StatusBadRequest, "invalid package name") - return - } - - packument, stale, err := h.servedPackument(r.Context(), packageName) - if err != nil { - if r.Context().Err() != nil { - return - } - if errors.Is(err, ErrUpstreamNotFound) || errors.Is(err, ErrPackageNotFound) { - JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("failed to read npm dist-tags", "package", packageName, "error", err) - JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") - return - } - - tags, err := jsonscan.Lookup(packument, "dist-tags") - if err != nil || len(tags) == 0 || tags[0] != '{' { - tags = []byte("{}") - } - markStale(w, stale) - w.Header().Set(headerContentType, contentTypeJSON) - w.WriteHeader(http.StatusOK) - _, _ = w.Write(tags) -} - -// servedPackument returns the packument of packageName as the proxy serves -// it: from the package's registry or route, filtered by cooldown and the -// denylist, with pushed versions added, or built from the pushed versions -// alone in a hosted-only scope. stale reports a cached copy served because -// upstream failed. -func (h *NPMHandler) servedPackument(ctx context.Context, packageName string) (body []byte, stale bool, err error) { - local, err := h.npmLocalEntries(packageName) - if err != nil { - return nil, false, err - } - if h.proxy.hostedOnly("npm", packageName) { - if len(local) == 0 { - return nil, false, ErrPackageNotFound - } - body, err := localNPMPackument(packageName, local) - return body, false, err - } - - registryURL, cacheKey := h.sourceFor(packageName) - _, registryPath := npmMetadataPath(packageName) - upstream := h.proxy.fetchMetadataDocument(ctx, "npm", cacheKey, registryURL+"/"+registryPath, h.packumentAccept(), h.packumentValidator()) - switch { - case errors.Is(upstream.err, ErrUpstreamNotFound) && len(local) > 0: - body, err := localNPMPackument(packageName, local) - return body, false, err - case upstream.err != nil: - return nil, false, upstream.err - case len(local) > 0: - body, _, err := h.rewriteMetadataWithLocal(packageName, upstream.body, local) - return body, upstream.stale, err - } - - rewritten, err := h.proxy.cachedRewrite(ctx, "npm", h.proxyURL, cacheKey, upstream.body, upstream.digest, func(b []byte) ([]byte, []string, error) { - return h.rewriteMetadataKeeping(packageName, b) - }) - if err != nil { - if h.proxy.mustFilterMetadata("npm", packageName) { - return nil, false, err - } - // The packument route serves upstream's document when the rewrite - // fails and nothing has to be filtered out. - return upstream.body, upstream.stale, nil - } - return rewritten.content(), upstream.stale, nil -} - -// validNPMPackageName reports whether name is a plain or scoped package -// name, without path traversal. -func validNPMPackageName(name string) bool { - if name == "" || containsPathTraversal(name) || strings.ContainsAny(name, "\\") { - return false - } - scope, rest, scoped := strings.Cut(name, "/") - if !scoped { - return !strings.HasPrefix(name, "@") - } - return len(scope) > 1 && scope[0] == '@' && rest != "" && !strings.Contains(rest, "/") -} diff --git a/internal/handler/npm_dist_tags_test.go b/internal/handler/npm_dist_tags_test.go deleted file mode 100644 index a23a1c9..0000000 --- a/internal/handler/npm_dist_tags_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package handler - -import ( - "encoding/json" - "net/http" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// taggedPackument has 1.0.0 published a month ago and 2.0.0 an hour ago, -// tagged latest. -func taggedPackument(name string) func(string) string { - month := time.Now().Add(-30 * 24 * time.Hour).UTC().Format(time.RFC3339) - hour := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) - return func(baseURL string) string { - return `{"name":"` + name + `","dist-tags":{"latest":"2.0.0","stable":"1.0.0"},"versions":{` + - `"1.0.0":{"version":"1.0.0","dist":{"tarball":"` + baseURL + `/x/-/x-1.0.0.tgz"}},` + - `"2.0.0":{"version":"2.0.0","dist":{"tarball":"` + baseURL + `/x/-/x-2.0.0.tgz"}}},` + - `"time":{"1.0.0":"` + month + `","2.0.0":"` + hour + `"}}` - } -} - -func decodeDistTags(t *testing.T, code int, body []byte) map[string]string { - t.Helper() - if code != http.StatusOK { - t.Fatalf("status = %d, body: %s", code, body) - } - var tags map[string]string - if err := json.Unmarshal(body, &tags); err != nil { - t.Fatalf("dist-tags are not a JSON object: %v: %s", err, body) - } - return tags -} - -func TestNPMDistTags(t *testing.T) { - upstream := newNPMUpstream(t, map[string]func(string) string{"/demo": taggedPackument("demo")}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - - w := serveNPM(t, h, http.MethodGet, "/-/package/demo/dist-tags", nil, nil) - tags := decodeDistTags(t, w.Code, w.Body.Bytes()) - if tags["latest"] != "2.0.0" || tags["stable"] != "1.0.0" || len(tags) != 2 { - t.Errorf("dist-tags = %v", tags) - } -} - -// TestNPMDistTagsFollowCooldown checks that latest never names a version -// cooldown holds back, as in the packument. -func TestNPMDistTagsFollowCooldown(t *testing.T) { - upstream := newNPMUpstream(t, map[string]func(string) string{"/demo": taggedPackument("demo")}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - proxy.Cooldown = &cooldown.Config{Default: "7d"} - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - - w := serveNPM(t, h, http.MethodGet, "/-/package/demo/dist-tags", nil, nil) - if tags := decodeDistTags(t, w.Code, w.Body.Bytes()); tags["latest"] != "1.0.0" { - t.Errorf("dist-tags = %v, want latest 1.0.0", tags) - } -} - -func TestNPMDistTagsRoutedPackage(t *testing.T) { - public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("9.9.9", "")}, http.StatusOK) - private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - h := newRoutedNPMHandler(proxy, public, private) - - w := serveNPM(t, h, http.MethodGet, "/-/package/@example%2fwidgets/dist-tags", nil, nil) - if tags := decodeDistTags(t, w.Code, w.Body.Bytes()); tags["latest"] != "1.0.0" { - t.Errorf("dist-tags = %v, want the route's latest 1.0.0", tags) - } - if got := public.requested(); len(got) != 0 { - t.Errorf("default registry was queried for a routed package: %v", got) - } -} - -func TestNPMDistTagsHostedOnlyScope(t *testing.T) { - f := newNPMHostedFixture(t, nil) - f.proxy.HostedNPMScopes = []string{"@acme"} - f.push(t, "@acme/private", "1.10.0", `{"name":"@acme/private","version":"1.10.0"}`) - f.push(t, "@acme/private", "1.9.0", `{"name":"@acme/private","version":"1.9.0"}`) - - rec := f.get(t, "/-/package/@acme%2fprivate/dist-tags") - if tags := decodeDistTags(t, rec.Code, rec.Body.Bytes()); tags["latest"] != "1.10.0" { - t.Errorf("dist-tags = %v, want latest 1.10.0", tags) - } - if rec := f.get(t, "/-/package/@acme%2fother/dist-tags"); rec.Code != http.StatusNotFound { - t.Errorf("unpushed package status = %d, want 404", rec.Code) - } - if *f.upstreamRequests != 0 { - t.Errorf("hosted-only scope sent %d requests upstream", *f.upstreamRequests) - } -} - -func TestNPMDistTagsErrors(t *testing.T) { - upstream := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - - tests := []struct { - method, target string - want int - }{ - {http.MethodGet, "/-/package/missing/dist-tags", http.StatusNotFound}, - {http.MethodGet, "/-/package/@scope/dist-tags", http.StatusBadRequest}, - {http.MethodGet, "/-/package/a/b/c/dist-tags", http.StatusBadRequest}, - {http.MethodGet, "/-/package/..%2f..%2fetc/dist-tags", http.StatusBadRequest}, - {http.MethodPut, "/-/package/demo/dist-tags/beta", http.StatusMethodNotAllowed}, - {http.MethodDelete, "/-/package/demo/dist-tags/beta", http.StatusMethodNotAllowed}, - } - for _, tt := range tests { - if w := serveNPM(t, h, tt.method, tt.target, nil, nil); w.Code != tt.want { - t.Errorf("%s %s: status = %d, want %d", tt.method, tt.target, w.Code, tt.want) - } - } - if got := upstream.requested(); len(got) != 1 { - t.Errorf("upstream requests = %v, want only the missing package", got) - } -} - -// TestNPMDistTagsWithPushedVersion checks that a push next to upstream -// versions leaves the tags as upstream has them, as in the packument. -func TestNPMDistTagsWithPushedVersion(t *testing.T) { - f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) - f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4"}`) - - rec := f.get(t, "/-/package/request/dist-tags") - if tags := decodeDistTags(t, rec.Code, rec.Body.Bytes()); tags["latest"] != "2.88.2" { - t.Errorf("dist-tags = %v, want upstream's latest 2.88.2", tags) - } -} diff --git a/internal/handler/npm_hosted.go b/internal/handler/npm_hosted.go deleted file mode 100644 index 77f9227..0000000 --- a/internal/handler/npm_hosted.go +++ /dev/null @@ -1,194 +0,0 @@ -package handler - -import ( - "encoding/json" - "errors" - "fmt" - "net/http" - "time" - - "github.com/Masterminds/semver/v3" - "github.com/git-pkgs/proxy/internal/metadata/npm" -) - -type npmLocalEntry = npm.LocalVersion - -// npmLocalEntries returns the pushed versions of a package as packument -// entries, or nil when there are none or hosted packages are disabled. -func (h *NPMHandler) npmLocalEntries(packageName string) ([]npmLocalEntry, error) { - local, err := h.proxy.localVersions("npm", packageName) - if err != nil || len(local) == 0 { - return nil, err - } - entries := make([]npmLocalEntry, 0, len(local)) - for _, lv := range local { - var manifest map[string]any - if err := json.Unmarshal(lv.Manifest, &manifest); err != nil { - return nil, fmt.Errorf("decoding manifest of %s@%s: %w", packageName, lv.Version, err) - } - manifest["name"] = packageName - manifest["version"] = lv.Version - manifest["_id"] = packageName + "@" + lv.Version - manifest["dist"] = map[string]string{ - "tarball": h.proxyTarballURL(packageName, lv.Version, lv.Filename), - "integrity": lv.Integrity, - } - entry, err := json.Marshal(manifest) - if err != nil { - return nil, err - } - entries = append(entries, npmLocalEntry{Version: lv.Version, Entry: entry, PublishedAt: lv.PushedAt.Format(time.RFC3339)}) - } - return entries, nil -} - -func findNPMLocalEntry(local []npmLocalEntry, version string) (npmLocalEntry, bool) { - for _, e := range local { - if e.Version == version { - return e, true - } - } - return npmLocalEntry{}, false -} - -// npmMetadataRequest is a packument or version metadata request. -type npmMetadataRequest struct { - packageName string - specifier string - versionLookup bool - upstreamURL string - accept string - metadataCacheKey string -} - -// serveLocalMetadata answers the metadata requests that pushed versions -// decide, and reports whether it did: a request for a pushed version, any -// request for a package in a hosted-only scope, and a packument of a package -// with pushed versions. Everything else takes the usual upstream path. -func (h *NPMHandler) serveLocalMetadata(w http.ResponseWriter, r *http.Request, req npmMetadataRequest) bool { - local, err := h.npmLocalEntries(req.packageName) - if err != nil { - h.proxy.Logger.Error("failed to read pushed npm versions", "package", req.packageName, "error", err) - JSONError(w, http.StatusInternalServerError, "failed to read pushed versions") - return true - } - switch e, pushed := findNPMLocalEntry(local, req.specifier); { - case req.versionLookup && pushed: - w.Header().Set(headerContentType, contentTypeJSON) - w.WriteHeader(http.StatusOK) - _, _ = w.Write(e.Entry) - case h.proxy.hostedOnly("npm", req.packageName): - h.serveHostedOnlyMetadata(w, r, req.packageName, req.specifier, req.versionLookup, req.metadataCacheKey, local) - case len(local) > 0 && !req.versionLookup: - h.serveMetadataWithLocal(w, r, req.packageName, req.upstreamURL, req.accept, req.metadataCacheKey, local) - default: - return false - } - return true -} - -// serveMetadataWithLocal serves a packument with the pushed versions of the -// package added. It bypasses the rewrite cache, whose entries are keyed by the -// upstream document alone, so a push or delete shows up on the next request. -// When the upstream does not know the package, the packument holds only the -// pushed versions. -func (h *NPMHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, accept, metadataCacheKey string, local []npmLocalEntry) { - upstream := h.proxy.fetchMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept, h.packumentValidator()) - var out []byte - var err error - switch { - case errors.Is(upstream.err, ErrUpstreamNotFound): - out, err = localNPMPackument(packageName, local) - case upstream.err != nil: - if writeUpstreamHeld(w, upstream.err) { - return - } - h.proxy.Logger.Error("failed to fetch npm metadata", "error", upstream.err) - JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") - return - default: - out, _, err = h.rewriteMetadataWithLocal(packageName, upstream.body, local) - } - if err != nil { - if errors.Is(err, ErrVersionDenied) { - JSONError(w, http.StatusForbidden, err.Error()) - return - } - h.proxy.Logger.Error("failed to add pushed versions to npm metadata", "package", packageName, "error", err) - JSONError(w, http.StatusBadGateway, "failed to rewrite package metadata") - return - } - markStale(w, upstream.stale) - serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("npm", h.proxyURL, metadataCacheKey, out)) -} - -// serveHostedOnlyMetadata answers a metadata request for a package in a -// hosted-only scope from its pushed versions alone. -func (h *NPMHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier string, versionLookup bool, metadataCacheKey string, local []npmLocalEntry) { - if len(local) == 0 { - JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) - return - } - if versionLookup { - if specifier == "latest" { - specifier = latestNPMVersion(local) - } - e, ok := findNPMLocalEntry(local, specifier) - if !ok { - JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) - return - } - w.Header().Set(headerContentType, contentTypeJSON) - w.WriteHeader(http.StatusOK) - _, _ = w.Write(e.Entry) - return - } - out, err := localNPMPackument(packageName, local) - if err != nil { - h.proxy.Logger.Error("failed to build npm metadata from pushed versions", "package", packageName, "error", err) - JSONError(w, http.StatusInternalServerError, "failed to build package metadata") - return - } - serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("npm", h.proxyURL, metadataCacheKey, out)) -} - -// localNPMPackument builds a packument from pushed versions alone. -func localNPMPackument(packageName string, local []npmLocalEntry) ([]byte, error) { - versions := make(map[string]json.RawMessage, len(local)) - times := make(map[string]string, len(local)) - for _, e := range local { - versions[e.Version] = e.Entry - times[e.Version] = e.PublishedAt - } - return json.Marshal(map[string]any{ - "name": packageName, - "dist-tags": map[string]string{"latest": latestNPMVersion(local)}, - "versions": versions, - "time": times, - }) -} - -// latestNPMVersion returns the highest pushed version, or the last pushed -// when a version is not semver. -func latestNPMVersion(local []npmLocalEntry) string { - latest, latestTime := "", "" - var latestSemver *semver.Version - semverOnly := true - for _, e := range local { - if e.PublishedAt > latestTime { - latest, latestTime = e.Version, e.PublishedAt - } - v, err := semver.NewVersion(e.Version) - if err != nil { - semverOnly = false - continue - } - if latestSemver == nil || v.GreaterThan(latestSemver) { - latestSemver = v - } - } - if semverOnly && latestSemver != nil { - return latestSemver.Original() - } - return latest -} diff --git a/internal/handler/npm_hosted_test.go b/internal/handler/npm_hosted_test.go deleted file mode 100644 index 083319a..0000000 --- a/internal/handler/npm_hosted_test.go +++ /dev/null @@ -1,369 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/registries/fetch" -) - -const requestPackument = `{ - "name": "request", - "dist-tags": {"latest": "2.88.2"}, - "versions": { - "2.88.2": {"name": "request", "version": "2.88.2", "dependencies": {"qs": "~6.5.1"}, - "dist": {"tarball": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", "integrity": "sha512-upstream2"}}, - "2.88.3": {"name": "request", "version": "2.88.3", "dependencies": {"old": "1.0.0"}, - "dist": {"tarball": "https://registry.npmjs.org/request/-/request-2.88.3.tgz", "integrity": "sha512-upstream3"}} - }, - "time": {"modified": "2020-02-11T00:00:00Z", "2.88.2": "2020-02-11T00:00:00Z", "2.88.3": "2020-02-12T00:00:00Z"} -}` - -type npmHostedFixture struct { - proxy *Proxy - handler http.Handler - upstreamRequests *int - fetcher *mockFetcher -} - -// newNPMHostedFixture serves packuments from upstream, keyed by request path, -// and answers 404 for anything else. -func newNPMHostedFixture(t *testing.T, upstream map[string]string) npmHostedFixture { - t.Helper() - upstreamRequests := 0 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamRequests++ - body, ok := upstream[r.URL.EscapedPath()] - if !ok { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(body)) - })) - t.Cleanup(srv.Close) - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HostedEnabled = true - proxy.HTTPClient = srv.Client() - return npmHostedFixture{proxy: proxy, handler: NewNPMHandler(proxy, "http://proxy.local", srv.URL).Routes(), - upstreamRequests: &upstreamRequests, fetcher: fetcher} -} - -func (f npmHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { - t.Helper() - target, err := HostedTargetFor("npm", name, version, "") - if err != nil { - t.Fatalf("HostedTargetFor: %v", err) - } - pushed, err := f.proxy.PushLocalArtifact(context.Background(), "npm", target, bytes.NewReader(npmTarball(manifest)), "", "") - if err != nil { - t.Fatalf("PushLocalArtifact: %v", err) - } - return pushed -} - -func (f npmHostedFixture) get(t *testing.T, path string) *httptest.ResponseRecorder { - t.Helper() - rec := httptest.NewRecorder() - f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) - return rec -} - -type npmTestPackument struct { - Name string `json:"name"` - DistTags map[string]string `json:"dist-tags"` - Versions map[string]npmTestVersion `json:"versions"` - Time map[string]string `json:"time"` - Extra map[string]json.RawMessage `json:"-"` -} - -type npmTestVersion struct { - Name string `json:"name"` - Version string `json:"version"` - ID string `json:"_id"` - Dependencies map[string]string `json:"dependencies"` - Dist struct { - Tarball string `json:"tarball"` - Integrity string `json:"integrity"` - } `json:"dist"` -} - -func decodePackument(t *testing.T, rec *httptest.ResponseRecorder) npmTestPackument { - t.Helper() - if rec.Code != http.StatusOK { - t.Fatalf("status = %d, body %s", rec.Code, rec.Body) - } - var doc npmTestPackument - if err := json.Unmarshal(rec.Body.Bytes(), &doc); err != nil { - t.Fatalf("decoding packument: %v\n%s", err, rec.Body) - } - return doc -} - -func TestNPMMetadataListsPushedVersion(t *testing.T) { - f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) - pushed := f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4","dependencies":{"qs":"~6.5.2"}}`) - - doc := decodePackument(t, f.get(t, "/request")) - - local, ok := doc.Versions["2.88.4"] - if !ok { - t.Fatalf("pushed version missing from versions: %v", doc.Versions) - } - if local.Name != "request" || local.Version != "2.88.4" || local.ID != "request@2.88.4" || local.Dependencies["qs"] != "~6.5.2" { - t.Errorf("pushed version entry = %+v", local) - } - if local.Dist.Tarball != "http://proxy.local/npm/request/-/request-2.88.4.tgz" { - t.Errorf("tarball = %q", local.Dist.Tarball) - } - if !strings.HasPrefix(local.Dist.Integrity, "sha512-") { - t.Errorf("integrity = %q, want the pushed archive's sha512", local.Dist.Integrity) - } - if doc.Time["2.88.4"] != pushed.PushedAt.Format(time.RFC3339) { - t.Errorf("time[2.88.4] = %q, want %q", doc.Time["2.88.4"], pushed.PushedAt.Format(time.RFC3339)) - } - if doc.Versions["2.88.2"].Dist.Tarball != "http://proxy.local/npm/request/-/request-2.88.2.tgz" { - t.Errorf("upstream version not rewritten: %+v", doc.Versions["2.88.2"]) - } - if doc.DistTags["latest"] != "2.88.2" { - t.Errorf("dist-tags.latest = %q, want upstream's 2.88.2", doc.DistTags["latest"]) - } -} - -func TestNPMMetadataPushedVersionReplacesUpstream(t *testing.T) { - f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) - f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3","dependencies":{"qs":"~6.5.2"}}`) - - rec := f.get(t, "/request") - if body := rec.Body.String(); strings.Contains(body, "sha512-upstream3") || strings.Count(body, `"2.88.3":{`) != 1 { - t.Fatalf("upstream entry for the pushed version still listed:\n%s", body) - } - local := decodePackument(t, rec).Versions["2.88.3"] - if local.Dependencies["qs"] != "~6.5.2" || local.Dependencies["old"] != "" { - t.Errorf("dependencies = %v, want the pushed package.json's", local.Dependencies) - } -} - -func TestNPMMetadataForPackageOnlyPushed(t *testing.T) { - f := newNPMHostedFixture(t, nil) - f.push(t, "@acme/private", "1.10.0", `{"name":"@acme/private","version":"1.10.0"}`) - f.push(t, "@acme/private", "1.9.0", `{"name":"@acme/private","version":"1.9.0"}`) - - doc := decodePackument(t, f.get(t, "/@acme/private")) - if doc.Name != "@acme/private" || len(doc.Versions) != 2 { - t.Errorf("packument = %+v", doc) - } - if doc.DistTags["latest"] != "1.10.0" { - t.Errorf("dist-tags.latest = %q, want the highest pushed version 1.10.0", doc.DistTags["latest"]) - } - if doc.Versions["1.9.0"].Dist.Tarball != "http://proxy.local/npm/@acme%2Fprivate/-/private-1.9.0.tgz" { - t.Errorf("tarball = %q", doc.Versions["1.9.0"].Dist.Tarball) - } - - if rec := f.get(t, "/@acme/other"); rec.Code != http.StatusNotFound { - t.Errorf("unknown package status = %d, want 404", rec.Code) - } -} - -func TestNPMVersionMetadataForPushedVersion(t *testing.T) { - f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) - f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4","dependencies":{"qs":"~6.5.2"}}`) - - rec := f.get(t, "/request/2.88.4") - if rec.Code != http.StatusOK { - t.Fatalf("status = %d, body %s", rec.Code, rec.Body) - } - var version npmTestVersion - if err := json.Unmarshal(rec.Body.Bytes(), &version); err != nil { - t.Fatalf("decoding version: %v", err) - } - if version.Version != "2.88.4" || version.Dist.Tarball != "http://proxy.local/npm/request/-/request-2.88.4.tgz" { - t.Errorf("version document = %+v", version) - } -} - -func TestNPMMetadataFollowsPushAndDelete(t *testing.T) { - f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) - f.proxy.CacheMetadata = true - f.proxy.MetadataTTL = time.Hour - f.proxy.SetMetadataRewriteCacheSize(1 << 20) - - before := f.get(t, "/request") - if before.Code != http.StatusOK { - t.Fatalf("status = %d", before.Code) - } - - f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4"}`) - pushed := f.get(t, "/request") - if _, ok := decodePackument(t, pushed).Versions["2.88.4"]; !ok { - t.Fatal("pushed version not listed while the upstream document is cached") - } - if pushed.Header().Get("ETag") == before.Header().Get("ETag") { - t.Error("ETag unchanged after a push") - } - - if err := f.proxy.DeleteLocalVersion("npm", "request", "2.88.4"); err != nil { - t.Fatalf("DeleteLocalVersion: %v", err) - } - if _, ok := decodePackument(t, f.get(t, "/request")).Versions["2.88.4"]; ok { - t.Error("deleted version still listed") - } -} - -func TestNPMMetadataIgnoresPushesWhenHostedDisabled(t *testing.T) { - f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) - f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4"}`) - f.proxy.HostedEnabled = false - - if _, ok := decodePackument(t, f.get(t, "/request")).Versions["2.88.4"]; ok { - t.Error("pushed version listed with hosted packages disabled") - } -} - -// With hosted packages disabled, a pushed archive is not served ahead of the -// upstream, but it is still never replaced by the upstream copy: the -// download is answered from its stored record like any cached archive. -func TestNPMDownloadWithHostedDisabledKeepsPush(t *testing.T) { - f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) - manifest := `{"name":"request","version":"2.88.3"}` - f.push(t, "request", "2.88.3", manifest) - f.proxy.HostedEnabled = false - - rec := f.get(t, "/request/-/request-2.88.3.tgz") - if rec.Code != http.StatusOK || rec.Body.String() != string(npmTarball(manifest)) { - t.Errorf("download status = %d, want 200 and the pushed tarball", rec.Code) - } - if f.fetcher.fetchCalled { - t.Error("the upstream tarball was fetched for a pushed version") - } -} - -func TestNPMDownloadOfPushedPackageWithCooldown(t *testing.T) { - f := newNPMHostedFixture(t, nil) - f.proxy.Cooldown = &cooldown.Config{Default: "7d"} - f.push(t, "@acme/private", "1.0.0", `{"name":"@acme/private","version":"1.0.0"}`) - - rec := f.get(t, "/@acme/private/-/private-1.0.0.tgz") - if rec.Code != http.StatusOK { - t.Fatalf("download status = %d, body %s, want the pushed tarball", rec.Code, rec.Body) - } - if _, ok := decodePackument(t, f.get(t, "/@acme/private")).Versions["1.0.0"]; !ok { - t.Error("pushed version missing from the packument with cooldown enabled") - } -} - -func TestNPMDownloadOfPushedOverrideOfVersionInCooldown(t *testing.T) { - recent := time.Now().Add(-time.Hour).Format(time.RFC3339) - packument := `{"name":"request","dist-tags":{"latest":"2.88.2"}, - "versions":{ - "2.88.2":{"name":"request","version":"2.88.2","dist":{"tarball":"https://registry.npmjs.org/request/-/request-2.88.2.tgz"}}, - "2.88.3":{"name":"request","version":"2.88.3","dist":{"tarball":"https://registry.npmjs.org/request/-/request-2.88.3.tgz"}}}, - "time":{"2.88.2":"2020-02-11T00:00:00Z","2.88.3":"` + recent + `"}}` - f := newNPMHostedFixture(t, map[string]string{"/request": packument}) - f.proxy.Cooldown = &cooldown.Config{Default: "7d"} - f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3"}`) - - if _, ok := decodePackument(t, f.get(t, "/request")).Versions["2.88.3"]; !ok { - t.Fatal("pushed version missing from the packument") - } - rec := f.get(t, "/request/-/request-2.88.3.tgz") - if rec.Code != http.StatusOK || rec.Body.String() != string(npmTarball(`{"name":"request","version":"2.88.3"}`)) { - t.Errorf("download status = %d, want 200 and the pushed tarball: the packument lists it, so the download must not be withheld", rec.Code) - } -} - -func TestNPMHostedOnlyScopeNeverAsksUpstream(t *testing.T) { - squatter := `{"name":"@acme/private","dist-tags":{"latest":"1.0.1"},"versions":{"1.0.1":{"name":"@acme/private","version":"1.0.1", - "dist":{"tarball":"https://registry.npmjs.org/@acme/private/-/private-1.0.1.tgz"}}}}` - f := newNPMHostedFixture(t, map[string]string{"/@acme%2fprivate": squatter, "/@acme%2Fprivate": squatter, "/request": requestPackument}) - f.proxy.HostedNPMScopes = []string{"@acme"} - f.push(t, "@acme/private", "1.0.0", `{"name":"@acme/private","version":"1.0.0"}`) - - doc := decodePackument(t, f.get(t, "/@acme/private")) - if len(doc.Versions) != 1 || doc.DistTags["latest"] != "1.0.0" { - t.Errorf("packument = %+v, want only the pushed 1.0.0", doc) - } - if rec := f.get(t, "/@acme/private/latest"); rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"version":"1.0.0"`) { - t.Errorf("latest status = %d, body %s", rec.Code, rec.Body) - } - if rec := f.get(t, "/@acme/private/1.0.1"); rec.Code != http.StatusNotFound { - t.Errorf("unpushed version status = %d, want 404", rec.Code) - } - if rec := f.get(t, "/@acme/private/-/private-1.0.1.tgz"); rec.Code != http.StatusNotFound { - t.Errorf("unpushed tarball status = %d, want 404", rec.Code) - } - if rec := f.get(t, "/@acme/other"); rec.Code != http.StatusNotFound { - t.Errorf("unpushed package status = %d, want 404", rec.Code) - } - if rec := f.get(t, "/@acme/private/-/private-1.0.0.tgz"); rec.Code != http.StatusOK { - t.Errorf("pushed tarball status = %d, want 200", rec.Code) - } - if *f.upstreamRequests != 0 { - t.Errorf("hosted-only scope sent %d requests upstream", *f.upstreamRequests) - } - - if rec := f.get(t, "/request"); rec.Code != http.StatusOK || *f.upstreamRequests == 0 { - t.Errorf("package outside the scope status = %d after %d upstream requests, want it fetched upstream", rec.Code, *f.upstreamRequests) - } -} - -func TestNPMDownloadOfUnreadablePushedTarball(t *testing.T) { - f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) - f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3"}`) - removeStored(t, f.proxy, "pkg:npm/request@2.88.3", "request-2.88.3.tgz") - - if rec := f.get(t, "/request/-/request-2.88.3.tgz"); rec.Code != http.StatusServiceUnavailable { - t.Errorf("download status = %d, body %s, want 503 rather than the upstream tarball", rec.Code, rec.Body) - } -} - -func TestNPMMetadataLeavesOutDeniedPushedVersions(t *testing.T) { - f := newNPMHostedFixture(t, nil) - f.push(t, "@acme/private", "1.0.0", `{"name":"@acme/private","version":"1.0.0"}`) - f.push(t, "@acme/private", "1.1.0", `{"name":"@acme/private","version":"1.1.0"}`) - setTestDenylist(t, f.proxy, "pkg:npm/@acme/private@1.1.0") - - doc := decodePackument(t, f.get(t, "/@acme/private")) - if _, ok := doc.Versions["1.1.0"]; ok || len(doc.Versions) != 1 { - t.Errorf("versions = %v, want the denied 1.1.0 left out", doc.Versions) - } - if doc.DistTags["latest"] != "1.0.0" { - t.Errorf("dist-tags.latest = %q, want 1.0.0", doc.DistTags["latest"]) - } -} - -func TestNPMDownloadOfPushedTarballWhileStreamingArtifacts(t *testing.T) { - const manifest = `{"name":"request","version":"2.88.3"}` - f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) - f.fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream tarball")), ContentType: "application/gzip"} - f.push(t, "request", "2.88.3", manifest) - f.proxy.StreamArtifacts = true - - rec := f.get(t, "/request/-/request-2.88.3.tgz") - if rec.Code != http.StatusOK || rec.Body.String() != string(npmTarball(manifest)) { - t.Errorf("download status = %d, body %q, want 200 and the pushed tarball", rec.Code, rec.Body) - } - if f.fetcher.fetchCalled { - t.Error("a pushed tarball was fetched upstream while streaming artifacts") - } -} - -func TestNPMDownloadOfUnreadablePushedTarballWhileStreamingArtifacts(t *testing.T) { - f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) - f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3"}`) - removeStored(t, f.proxy, "pkg:npm/request@2.88.3", "request-2.88.3.tgz") - f.proxy.StreamArtifacts = true - - if rec := f.get(t, "/request/-/request-2.88.3.tgz"); rec.Code != http.StatusServiceUnavailable { - t.Errorf("download status = %d, body %s, want 503 rather than the upstream tarball", rec.Code, rec.Body) - } -} diff --git a/internal/handler/npm_integrity_test.go b/internal/handler/npm_integrity_test.go deleted file mode 100644 index 2c14614..0000000 --- a/internal/handler/npm_integrity_test.go +++ /dev/null @@ -1,169 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha1" //nolint:gosec // npm's dist.shasum is SHA-1 - "encoding/hex" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/git-pkgs/registries/fetch" -) - -const npmIntegrityTarball = "tarball data" - -var npmIntegrityCases = []struct { - name string - dist map[string]string - wantOK bool -}{ - {"matching sha512", map[string]string{"integrity": sha512SRI(npmIntegrityTarball)}, true}, - {"mismatched sha512", map[string]string{"integrity": sha512SRI("other bytes")}, false}, - {"matching sha256", map[string]string{"integrity": sha256SRI(npmIntegrityTarball)}, true}, - {"strongest algorithm decides", map[string]string{"integrity": sha256SRI("other bytes") + " " + sha512SRI(npmIntegrityTarball)}, true}, - {"any digest of the strongest algorithm", map[string]string{"integrity": sha512SRI("other bytes") + " " + sha512SRI(npmIntegrityTarball)}, true}, - {"unparseable integrity", map[string]string{"integrity": "sha512-nope"}, true}, - {"matching shasum", map[string]string{"shasum": sha1Hex(npmIntegrityTarball)}, true}, - {"mismatched shasum", map[string]string{"shasum": sha1Hex("other bytes")}, false}, - {"integrity wins over shasum", map[string]string{"integrity": sha512SRI("other bytes"), "shasum": sha1Hex(npmIntegrityTarball)}, false}, - {"unparseable integrity falls back to shasum", map[string]string{"integrity": "sha512-nope", "shasum": sha1Hex("other bytes")}, false}, - {"unparseable shasum", map[string]string{"shasum": "not-a-sha1"}, true}, - {"no checksum", map[string]string{}, true}, -} - -// TestNPMDownloadVerifiesDeclaredIntegrity checks a freshly fetched tarball -// against dist.integrity: a mismatch is a 502 and nothing is cached. -func TestNPMDownloadVerifiesDeclaredIntegrity(t *testing.T) { - for _, tt := range npmIntegrityCases { - t.Run(tt.name, func(t *testing.T) { - proxy, srv := npmIntegrityProxy(t, tt.dist) - - resp, body := npmIntegrityGet(t, srv) - if wantStatus := map[bool]int{true: http.StatusOK, false: http.StatusBadGateway}[tt.wantOK]; resp.StatusCode != wantStatus { - t.Fatalf("status = %d, want %d; body: %s", resp.StatusCode, wantStatus, body) - } - if tt.wantOK && body != npmIntegrityTarball { - t.Errorf("body = %q, want the tarball", body) - } - cached, err := proxy.GetCachedArtifact(context.Background(), "npm", "leftpad", "1.0.0", "leftpad-1.0.0.tgz") - if err != nil { - t.Fatalf("reading cache: %v", err) - } - if cached != nil { - _ = cached.Reader.Close() - } - if (cached != nil) != tt.wantOK { - t.Errorf("cached = %v, want %v", cached != nil, tt.wantOK) - } - }) - } -} - -// TestNPMDownloadRecordsDeclaredIntegrity checks that the integrity a tarball -// was verified against is kept with the cached version, so cache hits are -// verified against it too. -func TestNPMDownloadRecordsDeclaredIntegrity(t *testing.T) { - sri := sha512SRI(npmIntegrityTarball) - proxy, srv := npmIntegrityProxy(t, map[string]string{"integrity": sri}) - if resp, body := npmIntegrityGet(t, srv); resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d; body: %s", resp.StatusCode, body) - } - - pkgPURL, versionPURL, err := packagePURLStrings("npm", "leftpad", "1.0.0") - if err != nil { - t.Fatal(err) - } - record, err := proxy.DB.GetCachedArtifact(pkgPURL, versionPURL, "leftpad-1.0.0.tgz") - if err != nil || record == nil { - t.Fatalf("cache record = %v, %v", record, err) - } - if record.Integrity.String != sri { - t.Errorf("recorded integrity = %q, want %q", record.Integrity.String, sri) - } -} - -// TestNPMStreamedDownloadVerifiesDeclaredIntegrity checks dist.integrity -// when tarballs are streamed without being stored: a mismatch aborts the -// response, so the client never receives a complete body. -func TestNPMStreamedDownloadVerifiesDeclaredIntegrity(t *testing.T) { - for _, tt := range npmIntegrityCases { - t.Run(tt.name, func(t *testing.T) { - proxy, srv := npmIntegrityProxy(t, tt.dist) - proxy.StreamArtifacts = true - - // The response is aborted on a mismatch: before its headers when - // the body fits in the server's buffer, during the body otherwise. - body, err := func() ([]byte, error) { - resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-1.0.0.tgz") - if err != nil { - return nil, err - } - defer func() { _ = resp.Body.Close() }() - return io.ReadAll(resp.Body) - }() - - if tt.wantOK { - if err != nil || string(body) != npmIntegrityTarball { - t.Errorf("body = %q, %v; want the complete tarball", body, err) - } - return - } - if err == nil { - t.Errorf("received the whole body (%q), want the response aborted", body) - } - }) - } -} - -func npmIntegrityProxy(t *testing.T, dist map[string]string) (*Proxy, *httptest.Server) { - t.Helper() - upstream := npmIntegrityUpstream(t, dist) - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader(npmIntegrityTarball)), Size: int64(len(npmIntegrityTarball))} - - srv := httptest.NewServer(NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) - t.Cleanup(srv.Close) - return proxy, srv -} - -func npmIntegrityGet(t *testing.T, srv *httptest.Server) (*http.Response, string) { - t.Helper() - resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-1.0.0.tgz") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - body, _ := io.ReadAll(resp.Body) - return resp, string(body) -} - -// npmIntegrityUpstream serves a leftpad packument whose 1.0.0 carries dist. -func npmIntegrityUpstream(t *testing.T, dist map[string]string) *httptest.Server { - t.Helper() - packument, err := json.Marshal(map[string]any{ - "name": "leftpad", - "dist-tags": map[string]string{"latest": "1.0.0"}, - "versions": map[string]any{ - "1.0.0": map[string]any{"dist": dist}, - }, - }) - if err != nil { - t.Fatalf("encoding packument: %v", err) - } - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", contentTypeJSON) - _, _ = w.Write(packument) - })) - t.Cleanup(upstream.Close) - return upstream -} - -func sha1Hex(data string) string { - sum := sha1.Sum([]byte(data)) //nolint:gosec // npm's dist.shasum is SHA-1 - return hex.EncodeToString(sum[:]) -} diff --git a/internal/handler/npm_packument_accept_test.go b/internal/handler/npm_packument_accept_test.go deleted file mode 100644 index 1a87fdd..0000000 --- a/internal/handler/npm_packument_accept_test.go +++ /dev/null @@ -1,210 +0,0 @@ -package handler - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/registries/fetch" -) - -// TestNPMTarballKeepsCachedPackumentFormat checks that a tarball download -// revalidates the packument in the format the metadata route cached it in, -// rather than replacing the abbreviated document every client is served with -// the full one. -func TestNPMTarballKeepsCachedPackumentFormat(t *testing.T) { - const abbreviated = "application/vnd.npm.install-v1+json" - var ( - mu sync.Mutex - accepts []string - ) - var upstream *httptest.Server - upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - accept := r.Header.Get("Accept") - mu.Lock() - accepts = append(accepts, accept) - mu.Unlock() - contentType, etag := contentTypeJSON, `"full"` - if strings.HasPrefix(accept, abbreviated) { - contentType, etag = abbreviated, `"abbreviated"` - } - w.Header().Set(headerETag, etag) - if r.Header.Get("If-None-Match") == etag { - w.WriteHeader(http.StatusNotModified) - return - } - w.Header().Set(headerContentType, contentType) - _, _ = fmt.Fprintf(w, `{"name":"demo","versions":{"1.0.0":{"name":"demo","version":"1.0.0","dist":{"tarball":%q}}}}`, - upstream.URL+"/demo/-/demo-1.0.0.tgz") - })) - t.Cleanup(upstream.Close) - - proxy, db, _, artifactFetcher := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Nanosecond // the tarball request revalidates - proxy.HTTPClient = upstream.Client() - artifactFetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package")), ContentType: "application/gzip"} - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - - if w := serveNPM(t, h, http.MethodGet, "/demo", nil, nil); w.Code != http.StatusOK { - t.Fatalf("metadata: status %d: %s", w.Code, w.Body.String()) - } - if w := serveNPM(t, h, http.MethodGet, "/demo/-/demo-1.0.0.tgz", nil, nil); w.Code != http.StatusOK { - t.Fatalf("tarball: status %d: %s", w.Code, w.Body.String()) - } - - mu.Lock() - defer mu.Unlock() - if len(accepts) != 2 { - t.Fatalf("upstream packument requests = %d (%q), want 2", len(accepts), accepts) - } - for _, accept := range accepts { - if accept != npmAcceptDefault { - t.Errorf("packument requested with Accept %q, want %q", accept, npmAcceptDefault) - } - } - entry, err := db.GetMetadataCache("npm", "demo") - if err != nil || entry == nil { - t.Fatalf("cache row: %v, %v", entry, err) - } - if entry.ContentType.String != abbreviated { - t.Errorf("cached packument Content-Type = %q, want %q", entry.ContentType.String, abbreviated) - } -} - -// abbreviatedThenFullRegistry serves a packument with a 2.0.0 published an -// hour ago and a 1.0.0 published a month ago. The abbreviated form, like -// registry.npmjs.org's, has no time map. fail makes requests for the full -// document fail. -func abbreviatedThenFullRegistry(t *testing.T, fail *atomic.Bool, fullRequests *atomic.Int64) *httptest.Server { - t.Helper() - const abbreviated = "application/vnd.npm.install-v1+json" - var upstream *httptest.Server - upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/demo" { - w.WriteHeader(http.StatusNotFound) - return - } - versions := fmt.Sprintf(`{"1.0.0":{"name":"demo","version":"1.0.0","dist":{"tarball":%q}},"2.0.0":{"name":"demo","version":"2.0.0","dist":{"tarball":%q}}}`, - upstream.URL+"/demo/-/demo-1.0.0.tgz", upstream.URL+"/demo/-/demo-2.0.0.tgz") - if strings.HasPrefix(r.Header.Get("Accept"), abbreviated) { - w.Header().Set(headerContentType, abbreviated) - _, _ = fmt.Fprintf(w, `{"name":"demo","modified":%q,"dist-tags":{"latest":"2.0.0"},"versions":%s}`, - time.Now().Format(time.RFC3339), versions) - return - } - fullRequests.Add(1) - if fail.Load() { - w.WriteHeader(http.StatusServiceUnavailable) - return - } - w.Header().Set(headerContentType, contentTypeJSON) - _, _ = fmt.Fprintf(w, `{"name":"demo","dist-tags":{"latest":"2.0.0"},"time":{"1.0.0":%q,"2.0.0":%q},"versions":%s}`, - time.Now().Add(-30*24*time.Hour).Format(time.RFC3339), time.Now().Add(-time.Hour).Format(time.RFC3339), versions) - })) - t.Cleanup(upstream.Close) - return upstream -} - -// seedAbbreviatedPackument caches the abbreviated packument for demo through -// the metadata route while cooldown is off, then turns cooldown on. -func seedAbbreviatedPackument(t *testing.T, upstream *httptest.Server) *NPMHandler { - t.Helper() - proxy, db, _, artifactFetcher := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - artifactFetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package")), ContentType: "application/gzip"} - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - - if w := serveNPM(t, h, http.MethodGet, "/demo", nil, nil); w.Code != http.StatusOK { - t.Fatalf("seeding metadata: status %d: %s", w.Code, w.Body.String()) - } - entry, err := db.GetMetadataCache("npm", "demo") - if err != nil || entry == nil || entry.ContentType.String != "application/vnd.npm.install-v1+json" { - t.Fatalf("expected an abbreviated packument in the cache, got %+v (%v)", entry, err) - } - - proxy.Cooldown = &cooldown.Config{Default: "7d"} - return h -} - -// A packument cached in abbreviated form before cooldown was turned on has -// no publish times, so cooldown could not filter anything while it stayed -// fresh. With cooldown on it must be refetched in full. -func TestNPMCooldownRefetchesCachedAbbreviatedPackument(t *testing.T) { - var fail atomic.Bool - var fullRequests atomic.Int64 - upstream := abbreviatedThenFullRegistry(t, &fail, &fullRequests) - h := seedAbbreviatedPackument(t, upstream) - - w := serveNPM(t, h, http.MethodGet, "/demo", nil, nil) - if w.Code != http.StatusOK { - t.Fatalf("metadata: status %d: %s", w.Code, w.Body.String()) - } - var doc struct { - DistTags map[string]string `json:"dist-tags"` - Versions map[string]json.RawMessage `json:"versions"` - } - if err := json.Unmarshal(w.Body.Bytes(), &doc); err != nil { - t.Fatalf("decoding metadata: %v", err) - } - if _, ok := doc.Versions["2.0.0"]; ok { - t.Error("version 2.0.0, published an hour ago, was not filtered by cooldown") - } - if _, ok := doc.Versions["1.0.0"]; !ok { - t.Error("version 1.0.0 missing") - } - if doc.DistTags["latest"] != "1.0.0" { - t.Errorf("dist-tags.latest = %q, want 1.0.0", doc.DistTags["latest"]) - } - if fullRequests.Load() == 0 { - t.Error("full packument was never fetched from upstream") - } - - if w := serveNPM(t, h, http.MethodGet, "/demo/-/demo-2.0.0.tgz", nil, nil); w.Code != http.StatusNotFound { - t.Errorf("tarball 2.0.0: status %d, want %d (withheld by cooldown)", w.Code, http.StatusNotFound) - } -} - -// The tarball check alone must not trust the abbreviated packument either. -func TestNPMCooldownTarballRefetchesCachedAbbreviatedPackument(t *testing.T) { - var fail atomic.Bool - var fullRequests atomic.Int64 - upstream := abbreviatedThenFullRegistry(t, &fail, &fullRequests) - h := seedAbbreviatedPackument(t, upstream) - - if w := serveNPM(t, h, http.MethodGet, "/demo/-/demo-2.0.0.tgz", nil, nil); w.Code != http.StatusNotFound { - t.Errorf("tarball 2.0.0: status %d, want %d (withheld by cooldown)", w.Code, http.StatusNotFound) - } - if fullRequests.Load() == 0 { - t.Error("full packument was never fetched from upstream") - } -} - -// When upstream cannot supply the full packument, the abbreviated one must not -// stand in for it as a stale copy: it would let every version through. -func TestNPMCooldownSkipsAbbreviatedStaleFallback(t *testing.T) { - var fail atomic.Bool - var fullRequests atomic.Int64 - upstream := abbreviatedThenFullRegistry(t, &fail, &fullRequests) - h := seedAbbreviatedPackument(t, upstream) - h.proxy.MetadataTTL = time.Nanosecond // force a refetch - fail.Store(true) - - w := serveNPM(t, h, http.MethodGet, "/demo", nil, nil) - if w.Code != http.StatusBadGateway { - t.Errorf("metadata: status %d, want %d; body: %s", w.Code, http.StatusBadGateway, w.Body.String()) - } - if strings.Contains(w.Body.String(), "2.0.0") { - t.Errorf("served the abbreviated packument as a stale copy: %s", w.Body.String()) - } -} diff --git a/internal/handler/npm_portable_test.go b/internal/handler/npm_portable_test.go deleted file mode 100644 index d0bb193..0000000 --- a/internal/handler/npm_portable_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package handler - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "log/slog" - "net/http" - "os" - "strings" - "testing" - - "github.com/git-pkgs/cooldown" -) - -func TestNPMMetadataLogsPolicyAndURLChanges(t *testing.T) { - body, err := os.ReadFile("../metadata/npm/testdata/packument.json") - if err != nil { - t.Fatal(err) - } - f := newNPMHostedFixture(t, map[string]string{"/@example%2Fwidgets": string(body)}) - f.proxy.Cooldown = &cooldown.Config{Default: "7d"} - setTestDenylist(t, f.proxy, "pkg:npm/@example/widgets@2.0.0") - var logs bytes.Buffer - f.proxy.Logger = slog.New(slog.NewJSONHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) - if rec := f.get(t, "/@example/widgets"); rec.Code != http.StatusOK { - t.Fatalf("HTTP %d: %s", rec.Code, rec.Body.String()) - } - want := map[string]map[string]string{ - "cooldown: filtering npm version": {"level": "INFO", "package": "@example/widgets", "version": "3.0.0", "published": "2099-01-01T00:00:00Z"}, - "rewrote tarball URLs": {"level": "DEBUG", "package": "@example/widgets", "count": "1"}, - } - decoder := json.NewDecoder(&logs) - for { - var record map[string]any - if err := decoder.Decode(&record); err == io.EOF { - break - } else if err != nil { - t.Fatal(err) - } - message, _ := record["msg"].(string) - fields, ok := want[message] - if !ok { - continue - } - for key, value := range fields { - if fmt.Sprint(record[key]) != value { - t.Errorf("%s: %s = %v, want %q", message, key, record[key], value) - } - } - delete(want, message) - } - if len(want) != 0 { - t.Errorf("missing log records: %v", want) - } -} - -func TestNPMRoutesWithPortableMetadata(t *testing.T) { - body, err := os.ReadFile("../metadata/npm/testdata/packument.json") - if err != nil { - t.Fatal(err) - } - for _, hosted := range []bool{false, true} { - name := "upstream" - if hosted { - name = "hosted override" - } - t.Run(name, func(t *testing.T) { - f := newNPMHostedFixture(t, map[string]string{"/@example%2Fwidgets": string(body)}) - f.proxy.Cooldown = &cooldown.Config{Default: "7d"} - setTestDenylist(t, f.proxy, "pkg:npm/@example/widgets@2.0.0") - if hosted { - f.push(t, "@example/widgets", "3.0.0", `{"name":"@example/widgets","version":"3.0.0","dependencies":{"local":"1.0.0"}}`) - } - rec := f.get(t, "/@example/widgets") - doc := decodePackument(t, rec) - count := 1 - if hosted { - count = 2 - } - if len(doc.Versions) != count || doc.DistTags["latest"] != "1.0.0" || len(doc.DistTags) != 1 { - t.Fatalf("filtered metadata: %s", rec.Body.String()) - } - if doc.Versions["1.0.0"].Dist.Tarball != "http://proxy.local/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz" || !strings.Contains(rec.Body.String(), "12345678901234567890") { - t.Fatalf("rewritten metadata: %s", rec.Body.String()) - } - if hosted && doc.Versions["3.0.0"].Dependencies["local"] != "1.0.0" { - t.Fatalf("hosted override: %s", rec.Body.String()) - } - checkNPMVersionResponses(t, f, hosted) - }) - } -} - -func checkNPMVersionResponses(t *testing.T, f npmHostedFixture, hosted bool) { - t.Helper() - for specifier, want := range map[string]int{"latest": http.StatusOK, "1.0.0": http.StatusOK, "2.0.0": http.StatusForbidden, "3.0.0": http.StatusNotFound} { - if hosted && specifier == "3.0.0" { - want = http.StatusOK - } - got := f.get(t, "/@example/widgets/"+specifier) - if got.Code != want { - t.Errorf("%s: HTTP %d, want %d: %s", specifier, got.Code, want, got.Body.String()) - } - } -} diff --git a/internal/handler/npm_rewrite_bench_test.go b/internal/handler/npm_rewrite_bench_test.go deleted file mode 100644 index cf8b895..0000000 --- a/internal/handler/npm_rewrite_bench_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package handler - -import ( - "testing" - - "github.com/git-pkgs/proxy/internal/metadata/npm/npmbench" -) - -func BenchmarkNPMRewriteMetadata(b *testing.B) { - body := npmbench.Packument(5000) - h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy.example"} - b.SetBytes(int64(len(body))) - b.ReportAllocs() - for b.Loop() { - if _, _, err := h.rewriteMetadataKeeping("big", body); err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkNPMVersionTarball(b *testing.B) { - body := npmbench.Packument(5000) - b.SetBytes(int64(len(body))) - b.ReportAllocs() - for b.Loop() { - if npmVersionTarball(body, "1.0.2500") == "" { - b.Fatal("tarball not found") - } - } -} diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go deleted file mode 100644 index 9b8e35b..0000000 --- a/internal/handler/npm_rewrite_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package handler - -import ( - "errors" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/proxy/internal/metrics" - "github.com/prometheus/client_golang/prometheus/testutil" -) - -// TestMetadataServesRewriteWithoutReadingStorage checks that a repeat -// request inside the metadata TTL is answered from the rewrite cache by the -// stored digest alone, without reading the cached document back. -func TestMetadataServesRewriteWithoutReadingStorage(t *testing.T) { - var upstreamCalls atomic.Int64 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamCalls.Add(1) - w.Header().Set("Content-Type", "application/json") - switch r.URL.Path { - case "/left-pad": - _, _ = w.Write([]byte(`{"name":"left-pad","versions":{"1.3.0":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"}}}}`)) - case "/p2/vendor/pkg.json": - _, _ = w.Write([]byte(`{"packages":{"vendor/pkg":[{"version":"1.0.0","dist":{"type":"zip","url":"https://example.com/pkg-1.0.0.zip"}}]}}`)) - default: - http.NotFound(w, r) - } - })) - defer upstream.Close() - - cases := []struct { - ecosystem, name, path string - handler func(*Proxy) http.Handler - }{ - {"npm", "left-pad", "/left-pad", func(p *Proxy) http.Handler { - return NewNPMHandler(p, "http://proxy.example", upstream.URL).Routes() - }}, - {"composer", "vendor/pkg", "/p2/vendor/pkg.json", func(p *Proxy) http.Handler { - return NewComposerHandlerWithUpstreams(p, "http://proxy.example", upstream.URL, upstream.URL).Routes() - }}, - } - for _, c := range cases { - t.Run(c.ecosystem, func(t *testing.T) { - upstreamCalls.Store(0) - proxy, db, store, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.SetMetadataRewriteCacheSize(1 << 20) - handler := c.handler(proxy) - - get := func() string { - rec := httptest.NewRecorder() - handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, c.path, nil)) - if rec.Code != http.StatusOK { - t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) - } - return rec.Body.String() - } - - first := get() - entry, err := db.GetMetadataCache(c.ecosystem, c.name) - if err != nil || entry == nil || !strings.HasPrefix(entry.ContentDigest.String, "sha256:") { - t.Fatalf("metadata row = %+v, %v; want a sha256 content digest", entry, err) - } - - store.mu.Lock() - store.openErr = errors.New("storage must not be read") - store.mu.Unlock() - - if second := get(); second != first { - t.Errorf("second response %q differs from first %q", second, first) - } - if got := upstreamCalls.Load(); got != 1 { - t.Errorf("upstream calls = %d, want 1", got) - } - }) - } -} - -// TestNPMStoredRewriteSkippedWhenStale checks that the digest fast path is -// only taken inside the metadata TTL. -func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) { - proxy, db, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.SetMetadataRewriteCacheSize(1 << 20) - - body := []byte(`{}`) - proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) - key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { - t.Fatal(err) - } - if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { - t.Fatal("fresh row: expected a stored rewrite") - } - - entry, _ := db.GetMetadataCache("npm", "demo") - entry.FetchedAt.Time = time.Now().Add(-2 * time.Hour) - if err := db.UpsertMetadataCache(entry); err != nil { - t.Fatal(err) - } - if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); ok { - t.Error("stale row: expected no stored rewrite") - } - - proxy.Cooldown = &cooldown.Config{Default: "3d"} - entry.FetchedAt.Time = time.Now() - _ = db.UpsertMetadataCache(entry) - if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { - t.Error("cooldown on, fresh row: expected the stored rewrite") - } -} - -// TestNPMStoredRewriteRecordsMetadataHit checks the digest fast path counts as -// a metadata cache hit with the bytes it serves, never as an artifact hit. -func TestNPMStoredRewriteRecordsMetadataHit(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.SetMetadataRewriteCacheSize(1 << 20) - - body := []byte(`{"name":"demo"}`) - proxy.cacheMetadataBlob(t.Context(), "npm", "hit-demo", metadataStoragePath("npm", "hit-demo"), &upstreamMetadata{body: body}) - key := rewriteCacheKey("npm", "http://proxy", "hit-demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { - t.Fatal(err) - } - - metadataHits := func() float64 { - return testutil.ToFloat64(metrics.CacheHits.WithLabelValues("npm", metrics.CacheKindMetadata)) - } - artifactHits := func() float64 { - return testutil.ToFloat64(metrics.CacheHits.WithLabelValues("npm", metrics.CacheKindArtifact)) - } - served := func() float64 { return testutil.ToFloat64(metrics.MetadataServedBytes.WithLabelValues("npm")) } - metadataBefore, artifactBefore, servedBefore := metadataHits(), artifactHits(), served() - - m, ok := proxy.storedRewrite("npm", "hit-demo", "http://proxy", "hit-demo") - if !ok { - t.Fatal("expected a stored rewrite") - } - - if diff := metadataHits() - metadataBefore; diff != 1 { - t.Errorf("metadata hits delta = %.0f, want 1", diff) - } - if diff := artifactHits() - artifactBefore; diff != 0 { - t.Errorf("artifact hits delta = %.0f, want 0", diff) - } - if diff := served() - servedBefore; diff != float64(len(m.content())) { - t.Errorf("metadata served bytes delta = %.0f, want %d", diff, len(m.content())) - } -} diff --git a/internal/handler/npm_routes_test.go b/internal/handler/npm_routes_test.go deleted file mode 100644 index 56fc952..0000000 --- a/internal/handler/npm_routes_test.go +++ /dev/null @@ -1,301 +0,0 @@ -package handler - -import ( - "io" - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/registries/fetch" -) - -// npmUpstreamRecorder serves canned packuments and records which paths were -// requested. -type npmUpstreamRecorder struct { - *httptest.Server - mu sync.Mutex - requests []string -} - -func newNPMUpstream(t *testing.T, packuments map[string]func(baseURL string) string, status int) *npmUpstreamRecorder { - t.Helper() - u := &npmUpstreamRecorder{} - u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - u.mu.Lock() - u.requests = append(u.requests, r.URL.Path) - u.mu.Unlock() - if status != http.StatusOK { - w.WriteHeader(status) - return - } - packument, ok := packuments[r.URL.Path] - if !ok { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, packument(u.URL)) - })) - t.Cleanup(u.Close) - return u -} - -func (u *npmUpstreamRecorder) requested() []string { - u.mu.Lock() - defer u.mu.Unlock() - return append([]string(nil), u.requests...) -} - -func routedTestPackument(version, tarballBase string) func(string) string { - return func(baseURL string) string { - return `{"name":"@example/widgets","dist-tags":{"latest":"` + version + `"},"versions":{"` + version + - `":{"version":"` + version + `","dist":{"tarball":"` + baseURL + tarballBase + `/@example/widgets/-/widgets-` + version + `.tgz"}}}}` - } -} - -const routedNPMPath = "/registry/@example/widgets" - -func newRoutedNPMHandler(proxy *Proxy, public, private *npmUpstreamRecorder) *NPMHandler { - return NewNPMHandler(proxy, "http://proxy.test", public.URL). - WithPackageRoutes(map[string]string{"@example/*": private.URL + "/registry"}) -} - -func TestNPMRoutedMetadataComesFromRouteOnly(t *testing.T) { - public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("9.9.9", "")}, http.StatusOK) - private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - h := newRoutedNPMHandler(proxy, public, private) - - w := serveNPM(t, h, http.MethodGet, "/@example%2fwidgets", nil, nil) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if got := public.requested(); len(got) != 0 { - t.Errorf("default registry was queried for a routed package: %v", got) - } - if strings.Contains(w.Body.String(), "9.9.9") { - t.Errorf("routed package served the default registry's metadata: %s", w.Body.String()) - } - wantTarball := "http://proxy.test/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz" - if !strings.Contains(w.Body.String(), wantTarball) { - t.Errorf("tarball not rewritten to %q: %s", wantTarball, w.Body.String()) - } -} - -func TestNPMRoutedMetadataDoesNotFallBack(t *testing.T) { - tests := []struct { - name string - privateStatus int - wantStatus int - }{ - {"package missing on route", http.StatusOK, http.StatusNotFound}, - {"route rejects credentials", http.StatusUnauthorized, http.StatusBadGateway}, - {"route unavailable", http.StatusInternalServerError, http.StatusBadGateway}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("9.9.9", "")}, http.StatusOK) - private := newNPMUpstream(t, map[string]func(string) string{}, tt.privateStatus) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - h := newRoutedNPMHandler(proxy, public, private) - - w := serveNPM(t, h, http.MethodGet, "/@example%2fwidgets", nil, nil) - - if w.Code != tt.wantStatus { - t.Errorf("status = %d, want %d", w.Code, tt.wantStatus) - } - if got := public.requested(); len(got) != 0 { - t.Errorf("default registry was queried for a routed package: %v", got) - } - }) - } -} - -func TestNPMRoutedMetadataIgnoresDefaultRegistryCache(t *testing.T) { - public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("9.9.9", "")}, http.StatusOK) - private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - - if w := serveNPM(t, NewNPMHandler(proxy, "http://proxy.test", public.URL), http.MethodGet, "/@example%2fwidgets", nil, nil); w.Code != http.StatusOK { - t.Fatalf("priming the default registry cache: status %d", w.Code) - } - - w := serveNPM(t, newRoutedNPMHandler(proxy, public, private), http.MethodGet, "/@example%2fwidgets", nil, nil) - - if strings.Contains(w.Body.String(), "9.9.9") { - t.Errorf("routed package served metadata cached from the default registry: %s", w.Body.String()) - } -} - -func TestNPMUnroutedMetadataUsesDefaultRegistry(t *testing.T) { - public := newNPMUpstream(t, map[string]func(string) string{"/lodash": routedTestPackument("4.17.21", "")}, http.StatusOK) - private := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - h := newRoutedNPMHandler(proxy, public, private) - - w := serveNPM(t, h, http.MethodGet, "/lodash", nil, nil) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) - } - if got := private.requested(); len(got) != 0 { - t.Errorf("route was queried for an unrouted package: %v", got) - } -} - -func TestNPMRoutedDownloadResolvesFromRoute(t *testing.T) { - public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("1.0.0", "")}, http.StatusOK) - private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) - proxy, _, _, artifactFetcher := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("package")), - ContentType: "application/gzip", - } - h := newRoutedNPMHandler(proxy, public, private) - - w := serveNPM(t, h, http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if want := private.URL + "/registry/@example/widgets/-/widgets-1.0.0.tgz"; artifactFetcher.fetchedURL != want { - t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) - } - if got := public.requested(); len(got) != 0 { - t.Errorf("default registry was queried for a routed download: %v", got) - } -} - -func TestNPMRoutedDownloadWithoutMetadataStaysOnRoute(t *testing.T) { - public := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) - private := newNPMUpstream(t, map[string]func(string) string{}, http.StatusInternalServerError) - proxy, _, _, artifactFetcher := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("package")), - ContentType: "application/gzip", - } - h := newRoutedNPMHandler(proxy, public, private) - - w := serveNPM(t, h, http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil) - - // Without the route's metadata the tarball can't be verified, so it is - // not fetched at all, and in particular not from the default registry. - if w.Code != http.StatusBadGateway { - t.Errorf("status = %d, want 502", w.Code) - } - if artifactFetcher.fetchedURL != "" { - t.Errorf("fetched URL = %q, want none", artifactFetcher.fetchedURL) - } -} - -func TestNPMRoutedDownloadIgnoresDefaultRegistryArtifact(t *testing.T) { - public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("1.0.0", "")}, http.StatusOK) - private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) - proxy, _, _, artifactFetcher := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("public package")), - ContentType: "application/gzip", - } - - if w := serveNPM(t, NewNPMHandler(proxy, "http://proxy.test", public.URL), http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil); w.Code != http.StatusOK { - t.Fatalf("priming the artifact cache: status %d; body: %s", w.Code, w.Body.String()) - } - - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("private package")), - ContentType: "application/gzip", - } - w := serveNPM(t, newRoutedNPMHandler(proxy, public, private), http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil) - - if w.Body.String() != "private package" { - t.Errorf("routed download served %q, want the routed registry's package", w.Body.String()) - } -} - -// timedTestPackument is routedTestPackument with a publish time for the -// version, published age ago. -func timedTestPackument(version, tarballBase string, age time.Duration) func(string) string { - published := time.Now().Add(-age).UTC().Format(time.RFC3339) - return func(baseURL string) string { - return `{"name":"@example/widgets","dist-tags":{"latest":"` + version + `"},"versions":{"` + version + - `":{"version":"` + version + `","dist":{"tarball":"` + baseURL + tarballBase + `/@example/widgets/-/widgets-` + version + `.tgz"}}},` + - `"time":{"` + version + `":"` + published + `"}}` - } -} - -// TestNPMRoutedCooldownUsesRoutePublishTimes is a regression test: the -// download cooldown check stored publish times under the package's plain -// version PURL, so a time read from one registry was trusted for the same -// version from another. A route's version published an hour ago must stay in -// a seven-day cooldown even when the default registry's version of the same -// name is a month old, whichever was downloaded first, and changing a route's -// upstream must not carry over the old one's times. -func TestNPMRoutedCooldownUsesRoutePublishTimes(t *testing.T) { - const tarball = "/@example/widgets/-/widgets-1.0.0.tgz" - month, hour := 30*24*time.Hour, time.Hour - - type step struct { - handler string // "public", "route" or "other route" - wantStatus int - } - tests := []struct { - name string - steps []step - }{ - {"route added after a public download", []step{ - {"public", http.StatusOK}, - {"route", http.StatusNotFound}, - }}, - {"public download after a routed one", []step{ - {"route", http.StatusNotFound}, - {"public", http.StatusOK}, - }}, - {"route moved to another upstream", []step{ - {"other route", http.StatusOK}, - {"route", http.StatusNotFound}, - }}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": timedTestPackument("1.0.0", "", month)}, http.StatusOK) - private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: timedTestPackument("1.0.0", "/registry", hour)}, http.StatusOK) - old := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: timedTestPackument("1.0.0", "/registry", month)}, http.StatusOK) - proxy, _, _, artifactFetcher := setupTestProxy(t) - proxy.HTTPClient = http.DefaultClient - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.Cooldown = &cooldown.Config{Default: "7d"} - handlers := map[string]*NPMHandler{ - "public": NewNPMHandler(proxy, "http://proxy.test", public.URL), - "route": newRoutedNPMHandler(proxy, public, private), - "other route": newRoutedNPMHandler(proxy, public, old), - } - - for _, s := range tt.steps { - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("package")), - ContentType: "application/gzip", - } - w := serveNPM(t, handlers[s.handler], http.MethodGet, tarball, nil, nil) - if w.Code != s.wantStatus { - t.Errorf("%s download: status %d, want %d; body: %s", s.handler, w.Code, s.wantStatus, w.Body.String()) - } - } - }) - } -} diff --git a/internal/handler/npm_stream_harden_test.go b/internal/handler/npm_stream_harden_test.go deleted file mode 100644 index 0cebf0a..0000000 --- a/internal/handler/npm_stream_harden_test.go +++ /dev/null @@ -1,238 +0,0 @@ -package handler - -import ( - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - "time" -) - -// bigMetadataFiles returns a copy of what is in storage under the metadata path -// of big, staged copies included. -func bigMetadataFiles(store *mockStorage) map[string]string { - store.mu.Lock() - defer store.mu.Unlock() - files := map[string]string{} - for path, b := range store.files { - if strings.HasPrefix(path, metadataStoragePath("npm", "big")) { - files[path] = string(b) - } - } - return files -} - -// TestNPMStreamedPackumentBlobCorrupt checks that a stored copy whose bytes -// no longer match the digest of its row, within its TTL, is fetched again -// rather than failing the request, also above metadata_max_size. -func TestNPMStreamedPackumentBlobCorrupt(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) - proxy, store, handler := streamProxy(t, u) - proxy.MetadataMaxSize = int64(len(body) / 4) - decodeStreamed(t, serve(handler, "/big", nil)) - - path := metadataStoragePath("npm", "big") - store.mu.Lock() - store.files[path] = []byte(strings.Replace(body, "1.0.5", "1.0.X", 1)) - store.mu.Unlock() - proxy.SetMetadataRewriteCacheSize(64 << 20) - - if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { - t.Errorf("served %d versions, want 400", len(doc.Versions)) - } - if got := bigMetadataFiles(store)[path]; got != body { - t.Error("the corrupt copy was not replaced") - } -} - -// TestNPMStreamedPackumentTruncated checks that a document upstream cuts -// off halfway neither replaces the stored copy, which is served instead, -// nor leaves anything behind, with and without cooldown. -func TestNPMStreamedPackumentTruncated(t *testing.T) { - for _, withCooldown := range []bool{true, false} { - t.Run(map[bool]string{true: "cooldown", false: "no cooldown"}[withCooldown], func(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) - proxy, store, handler := streamProxy(t, u) - if !withCooldown { - proxy.Cooldown = nil - } - proxy.MetadataMaxSize = int64(len(body) / 4) - want := len(decodeStreamed(t, serve(handler, "/big", nil)).Versions) - before := bigMetadataFiles(store) - - u.replace(bigPackument(401), `"v2"`) - u.truncate.Store(true) - proxy.MetadataTTL = time.Nanosecond - proxy.SetMetadataRewriteCacheSize(64 << 20) - - if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != want { - t.Errorf("served %d versions, want the stored %d", len(doc.Versions), want) - } - after := bigMetadataFiles(store) - if len(after) != len(before) { - t.Errorf("storage holds %d files, %d before", len(after), len(before)) - } - for path, b := range before { - if after[path] != b { - t.Errorf("%s was changed", path) - } - } - }) - } -} - -// TestNPMStreamedPackumentTruncatedUncached checks that a document upstream -// cuts off halfway fails the request and is not kept. -func TestNPMStreamedPackumentTruncatedUncached(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) - proxy, store, handler := streamProxy(t, u) - proxy.Cooldown = nil - proxy.MetadataMaxSize = int64(len(body) / 4) - u.truncate.Store(true) - - if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { - t.Errorf("status %d, want 502", rec.Code) - } - if files := bigMetadataFiles(store); len(files) != 0 { - t.Errorf("left %d files in storage", len(files)) - } - if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { - t.Error("left a metadata cache row") - } -} - -// TestNPMStreamedPackumentUncachedErrors checks the answers to upstream -// failures with nothing cached. -func TestNPMStreamedPackumentUncachedErrors(t *testing.T) { - for _, tt := range []struct { - name string - set func(*streamUpstream) - want int - }{ - {"not found", func(u *streamUpstream) { u.gone.Store(true) }, http.StatusNotFound}, - {"upstream down", func(u *streamUpstream) { u.fail.Store(true) }, http.StatusBadGateway}, - } { - t.Run(tt.name, func(t *testing.T) { - u := newStreamUpstream(t, bigPackument(10)) - _, store, handler := streamProxy(t, u) - tt.set(u) - if rec := serve(handler, "/big", nil); rec.Code != tt.want { - t.Errorf("status %d, want %d", rec.Code, tt.want) - } - if files := bigMetadataFiles(store); len(files) != 0 { - t.Errorf("left %d files in storage", len(files)) - } - }) - } -} - -// TestNPMStreamedPackumentConcurrent checks that concurrent requests for an -// uncached packument share one upstream fetch and all get it. -func TestNPMStreamedPackumentConcurrent(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) - proxy, _, handler := streamProxy(t, u) - proxy.MetadataMaxSize = int64(len(body) / 4) - const n = 16 - var wg sync.WaitGroup - recs := make([]*httptest.ResponseRecorder, n) - for i := range n { - wg.Go(func() { recs[i] = serve(handler, "/big", nil) }) - } - wg.Wait() - for _, rec := range recs { - if doc := decodeStreamed(t, rec); len(doc.Versions) != 400 { - t.Errorf("served %d versions, want 400", len(doc.Versions)) - } - } - if got := u.sent.Load(); got != 1 { - t.Errorf("upstream sent the document %d times, want once", got) - } -} - -// TestNPMStreamedPackumentClientRevalidates checks that a client sending -// the ETag it got gets a 304, and a new ETag once the document changes. -func TestNPMStreamedPackumentClientRevalidates(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) - proxy, _, handler := streamProxy(t, u) - proxy.MetadataMaxSize = int64(len(body) / 4) - - first := serve(handler, "/big", nil) - etag := first.Header().Get("ETag") - if first.Code != http.StatusOK || etag == "" { - t.Fatalf("status %d, ETag %q", first.Code, etag) - } - if rec := serve(handler, "/big", http.Header{"If-None-Match": {etag}}); rec.Code != http.StatusNotModified { - t.Errorf("same document: status %d, want 304", rec.Code) - } - - u.replace(bigPackument(401), `"v2"`) - proxy.MetadataTTL = time.Nanosecond - rec := serve(handler, "/big", http.Header{"If-None-Match": {etag}}) - if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { - t.Errorf("changed document: status %d, ETag %q (was %q)", rec.Code, rec.Header().Get("ETag"), etag) - } - if doc := decodeStreamed(t, rec); len(doc.Versions) != 401 { - t.Errorf("served %d versions, want 401", len(doc.Versions)) - } -} - -// TestNPMStreamedPackumentOverStreamLimit checks that a document over -// MetadataStreamMaxSize is neither stored nor served: the stored copy is -// served instead, with and without cooldown, and with nothing stored the -// request fails. -func TestNPMStreamedPackumentOverStreamLimit(t *testing.T) { - for _, withCooldown := range []bool{true, false} { - t.Run(map[bool]string{true: "cooldown", false: "no cooldown"}[withCooldown], func(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) - proxy, store, handler := streamProxy(t, u) - if !withCooldown { - proxy.Cooldown = nil - } - proxy.MetadataMaxSize = int64(len(body) / 4) - proxy.MetadataStreamMaxSize = int64(len(body) + 1000) - want := len(decodeStreamed(t, serve(handler, "/big", nil)).Versions) - before := bigMetadataFiles(store) - - u.replace(bigPackument(500), `"v2"`) - proxy.MetadataTTL = time.Nanosecond - proxy.SetMetadataRewriteCacheSize(64 << 20) - - if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != want { - t.Errorf("served %d versions, want the stored %d", len(doc.Versions), want) - } - after := bigMetadataFiles(store) - if len(after) != len(before) { - t.Errorf("storage holds %d files, %d before", len(after), len(before)) - } - for path, b := range before { - if after[path] != b { - t.Errorf("%s was changed", path) - } - } - }) - } - t.Run("uncached", func(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) - proxy, store, handler := streamProxy(t, u) - proxy.MetadataMaxSize = int64(len(body) / 4) - proxy.MetadataStreamMaxSize = int64(len(body) / 2) - - if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { - t.Errorf("status %d, want 502", rec.Code) - } - if files := bigMetadataFiles(store); len(files) != 0 { - t.Errorf("left %d files in storage", len(files)) - } - if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { - t.Error("left a metadata cache row") - } - }) -} diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go deleted file mode 100644 index 5d2ab73..0000000 --- a/internal/handler/npm_stream_test.go +++ /dev/null @@ -1,464 +0,0 @@ -package handler - -import ( - "bytes" - "compress/gzip" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/registries/fetch" -) - -// bigPackument returns a packument laid out as registry.npmjs.org writes -// them, dist-tags before versions and time after, with n old versions padded -// so the document is large, and a latest published an hour ago. -func bigPackument(n int) string { - return bigPackumentAt("https://registry.npmjs.org", n) -} - -// bigPackumentAt is bigPackument with tarballs under base. -func bigPackumentAt(base string, n int) string { - var b strings.Builder - b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"9.0.0","old":"1.0.0"},"versions":{`) - times := []string{`"created":"2015-01-01T00:00:00Z"`} - for i := range n { - v := fmt.Sprintf("1.0.%d", i) - if i == 0 { - v = "1.0.0" - } - fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"readme":%q,"dist":{"tarball":"%s/big/-/big-%s.tgz"}},`, - v, v, strings.Repeat("x", 512), base, v) - times = append(times, fmt.Sprintf(`%q:"2016-01-01T00:%02d:%02dZ"`, v, i/60%60, i%60)) - } - recent := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) - b.WriteString(`"9.0.0":{"name":"big","version":"9.0.0","dist":{"tarball":"` + base + `/big/-/big-9.0.0.tgz"}}},`) - times = append(times, `"9.0.0":"`+recent+`"`) - b.WriteString(`"time":{` + strings.Join(times, ",") + `},"readme":"big"}`) - return b.String() -} - -type streamUpstream struct { - *httptest.Server - body atomic.Value // string - etag atomic.Value // string - fail atomic.Bool - gone atomic.Bool - // truncate makes upstream close the connection halfway through the - // document it announced. - truncate atomic.Bool - requests atomic.Int32 - // sent counts the responses that carried the document. - sent atomic.Int32 -} - -// replace makes upstream send body under a new ETag, so a revalidation -// gets it in a 200. -func (u *streamUpstream) replace(body, etag string) { - u.body.Store(body) - u.etag.Store(etag) -} - -func newStreamUpstream(t *testing.T, body string) *streamUpstream { - t.Helper() - u := &streamUpstream{} - u.replace(body, `"v1"`) - u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - u.requests.Add(1) - switch { - case u.fail.Load(): - http.Error(w, "down", http.StatusBadGateway) - case u.gone.Load(): - http.NotFound(w, r) - case r.URL.Path == "/big" && r.Header.Get("If-None-Match") == u.etag.Load().(string): - w.WriteHeader(http.StatusNotModified) - case r.URL.Path == "/big": - u.sent.Add(1) - body := u.body.Load().(string) - w.Header().Set("Content-Type", "application/json") - w.Header().Set("ETag", u.etag.Load().(string)) - if u.truncate.Load() { - w.Header().Set("Content-Length", fmt.Sprint(len(body))) - _, _ = io.WriteString(w, body[:len(body)/2]) - return - } - _, _ = io.WriteString(w, body) - default: - http.NotFound(w, r) - } - })) - t.Cleanup(u.Close) - return u -} - -func streamProxy(t *testing.T, u *streamUpstream) (*Proxy, *mockStorage, http.Handler) { - t.Helper() - proxy, _, store, _ := setupTestProxy(t) - proxy.HTTPClient = u.Client() - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.SetMetadataRewriteCacheSize(64 << 20) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - return proxy, store, NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() -} - -type packumentDoc struct { - DistTags map[string]string `json:"dist-tags"` - Versions map[string]json.RawMessage `json:"versions"` - Time map[string]string `json:"time"` -} - -func decodeStreamed(t *testing.T, rec *httptest.ResponseRecorder) packumentDoc { - t.Helper() - if rec.Code != http.StatusOK { - t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) - } - body := rec.Body.Bytes() - if rec.Header().Get("Content-Encoding") == "gzip" { - zr, err := gzip.NewReader(bytes.NewReader(body)) - if err != nil { - t.Fatal(err) - } - if body, err = io.ReadAll(zr); err != nil { - t.Fatal(err) - } - } - var doc packumentDoc - if err := json.Unmarshal(body, &doc); err != nil { - t.Fatalf("response is not JSON: %v", err) - } - return doc -} - -// TestNPMStreamedPackument requests a packument larger than -// metadata_max_size with cooldown on, its time map after its versions and -// its latest inside cooldown. The in-memory rewrite can't read it at all. -func TestNPMStreamedPackument(t *testing.T) { - const versions = 400 - body := bigPackument(versions) - u := newStreamUpstream(t, body) - proxy, _, handler := streamProxy(t, u) - proxy.MetadataMaxSize = int64(len(body) / 4) - - for _, header := range []http.Header{nil, {"Accept-Encoding": {"gzip"}}} { - doc := decodeStreamed(t, serve(handler, "/big", header)) - if len(doc.Versions) != versions { - t.Errorf("%d versions, want %d without 9.0.0", len(doc.Versions), versions) - } - if _, ok := doc.Versions["9.0.0"]; ok { - t.Error("9.0.0 is listed although it is inside cooldown") - } - if _, ok := doc.Time["9.0.0"]; ok { - t.Error("time still lists 9.0.0") - } - if doc.DistTags["latest"] != fmt.Sprintf("1.0.%d", versions-1) || doc.DistTags["old"] != "1.0.0" { - t.Errorf("dist-tags = %v, want latest moved to the newest version left", doc.DistTags) - } - if !strings.Contains(string(doc.Versions["1.0.0"]), `"tarball":"http://proxy.example/npm/big/-/big-1.0.0.tgz"`) { - t.Errorf("tarball not rewritten: %s", doc.Versions["1.0.0"]) - } - } - if n := u.requests.Load(); n != 1 { - t.Errorf("upstream asked %d times, want once", n) - } - - // Without the rewrite cache the in-memory path runs, which the size - // limit stops. - proxy.SetMetadataRewriteCacheSize(0) - if rec := serve(handler, "/big", nil); rec.Code == http.StatusOK { - t.Error("in-memory rewrite served a packument over metadata_max_size; the test no longer covers the stream") - } -} - -// TestNPMStreamedPackumentMatchesInMemory checks that both paths serve the -// same document and ETag. -func TestNPMStreamedPackumentMatchesInMemory(t *testing.T) { - body := bigPackument(50) - u := newStreamUpstream(t, body) - _, _, streamed := streamProxy(t, u) - inMemory, _, handler := streamProxy(t, u) - inMemory.CacheMetadata = false - - a, b := serve(streamed, "/big", nil), serve(handler, "/big", nil) - if a.Code != http.StatusOK || b.Code != http.StatusOK { - t.Fatalf("status %d and %d", a.Code, b.Code) - } - var ca, cb bytes.Buffer - if err := json.Compact(&ca, a.Body.Bytes()); err != nil { - t.Fatal(err) - } - if err := json.Compact(&cb, b.Body.Bytes()); err != nil { - t.Fatal(err) - } - if !bytes.Equal(ca.Bytes(), cb.Bytes()) { - t.Error("streamed and in-memory rewrites differ") - } -} - -// TestNPMStreamedPackumentStale serves the stored copy when upstream fails -// after the TTL, and 404 for a package upstream doesn't have. -func TestNPMStreamedPackumentStale(t *testing.T) { - u := newStreamUpstream(t, bigPackument(10)) - proxy, _, handler := streamProxy(t, u) - decodeStreamed(t, serve(handler, "/big", nil)) - - proxy.MetadataTTL = time.Nanosecond - proxy.SetMetadataRewriteCacheSize(64 << 20) // drop the cached rewrite - u.fail.Store(true) - if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { - t.Errorf("stale copy has %d versions, want 10", len(doc.Versions)) - } - - u.fail.Store(false) - if rec := serve(handler, "/missing", nil); rec.Code != http.StatusNotFound { - t.Errorf("missing package: status %d, want 404", rec.Code) - } -} - -// TestNPMStreamedPackumentStaleOn404 checks that a stored copy past its TTL -// is served when upstream answers 404, as the in-memory path does. -func TestNPMStreamedPackumentStaleOn404(t *testing.T) { - u := newStreamUpstream(t, bigPackument(10)) - proxy, _, handler := streamProxy(t, u) - decodeStreamed(t, serve(handler, "/big", nil)) - - proxy.MetadataTTL = time.Nanosecond - u.gone.Store(true) - for _, size := range []int64{64 << 20, 0} { // streamed, then in memory - proxy.SetMetadataRewriteCacheSize(size) - if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { - t.Errorf("rewrite cache %d: stale copy has %d versions, want 10", size, len(doc.Versions)) - } - } -} - -// TestNPMStreamedPackumentBlobMissing checks that a row whose document is -// gone from storage, within its TTL or revalidated by a 304 after it, fetches -// the document again, also when it is larger than metadata_max_size. -func TestNPMStreamedPackumentBlobMissing(t *testing.T) { - for _, tt := range []struct { - name string - ttl time.Duration - }{ - {"fresh row", time.Hour}, - {"expired row", time.Nanosecond}, - } { - t.Run(tt.name, func(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) - proxy, store, handler := streamProxy(t, u) - proxy.MetadataMaxSize = int64(len(body) / 4) - decodeStreamed(t, serve(handler, "/big", nil)) - - path := metadataStoragePath("npm", "big") - store.mu.Lock() - delete(store.files, path) - store.mu.Unlock() - proxy.MetadataTTL = tt.ttl - proxy.SetMetadataRewriteCacheSize(64 << 20) - - if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { - t.Errorf("served %d versions, want 400", len(doc.Versions)) - } - if n := u.sent.Load(); n != 2 { - t.Errorf("upstream sent the document %d times, want 2", n) - } - store.mu.Lock() - defer store.mu.Unlock() - if string(store.files[path]) != body { - t.Error("the document was not stored again") - } - }) - } -} - -// TestNPMStreamedVersionBlobMissing checks that a version request and a -// tarball download fetch a packument again whose stored copy is gone while -// its row is within the TTL. -func TestNPMStreamedVersionBlobMissing(t *testing.T) { - u := newStreamUpstream(t, "") - u.body.Store(bigPackumentAt(u.URL, 100)) - proxy, store, handler := streamProxy(t, u) - proxy.MetadataMaxSize = 1024 - fetcher := proxy.Fetcher.(*mockFetcher) - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} - path := metadataStoragePath("npm", "big") - - for _, req := range []string{"/big/1.0.5", "/big/-/big-1.0.5.tgz"} { - decodeStreamed(t, serve(handler, "/big", nil)) - store.mu.Lock() - delete(store.files, path) - store.mu.Unlock() - proxy.SetMetadataRewriteCacheSize(64 << 20) - - if rec := serve(handler, req, nil); rec.Code != http.StatusOK { - t.Errorf("%s: status %d: %.200s", req, rec.Code, rec.Body.String()) - } - store.mu.Lock() - _, ok := store.files[path] - store.mu.Unlock() - if !ok { - t.Errorf("%s: the packument was not stored again", req) - } - } -} - -// TestNPMStreamedPackumentChangedInStorage checks that a document replaced -// in storage between the two passes isn't rewritten from mixed bytes. -func TestNPMStreamedPackumentChangedInStorage(t *testing.T) { - u := newStreamUpstream(t, bigPackument(10)) - proxy, store, _ := streamProxy(t, u) - entry, err := proxy.storedMetadataDocument(context.Background(), "npm", "big", u.URL+"/big", contentTypeJSON, nil) - if err != nil { - t.Fatal(err) - } - store.mu.Lock() - store.files[entry.StoragePath] = []byte(bigPackument(11)) - store.mu.Unlock() - - r, err := proxy.openStoredMetadata(context.Background(), entry) - if err != nil { - t.Fatal(err) - } - defer func() { _ = r.Close() }() - if _, err := io.ReadAll(r); !errors.Is(err, errStoredMetadataChanged) { - t.Errorf("read error %v, want errStoredMetadataChanged", err) - } -} - -// TestNPMStreamedPackumentRevalidates checks that a 304 after the TTL keeps -// the stored document and its cached rewrite, without storing it again. -func TestNPMStreamedPackumentRevalidates(t *testing.T) { - u := newStreamUpstream(t, bigPackument(10)) - proxy, store, handler := streamProxy(t, u) - first := serve(handler, "/big", nil) - decodeStreamed(t, first) - store.mu.Lock() - stores := store.stores - store.mu.Unlock() - - proxy.MetadataTTL = time.Nanosecond - second := serve(handler, "/big", nil) - decodeStreamed(t, second) - if second.Header().Get("ETag") != first.Header().Get("ETag") { - t.Errorf("ETag changed from %s to %s", first.Header().Get("ETag"), second.Header().Get("ETag")) - } - if n := u.requests.Load(); n != 2 { - t.Errorf("upstream asked %d times, want 2", n) - } - store.mu.Lock() - defer store.mu.Unlock() - if store.stores != stores { - t.Errorf("the 304 stored the document again (%d stores, %d before)", store.stores, stores) - } -} - -// TestNPMStreamedPackumentKeepsFullCopy checks that an abbreviated packument -// from upstream, where cooldown needs the full one, doesn't replace the full -// copy in storage, which is served instead as when upstream fails. -func TestNPMStreamedPackumentKeepsFullCopy(t *testing.T) { - full := bigPackument(10) - u := newStreamUpstream(t, full) - proxy, store, handler := streamProxy(t, u) - decodeStreamed(t, serve(handler, "/big", nil)) - - proxy.MetadataTTL = time.Nanosecond - proxy.SetMetadataRewriteCacheSize(64 << 20) - u.replace(`{"name":"big","modified":"2026-01-01T00:00:00Z","dist-tags":{"latest":"1.0.0"},"versions":{"1.0.0":{}}}`, `"v2"`) - if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { - t.Errorf("served %d versions, want the 10 of the full copy", len(doc.Versions)) - } - if n := u.sent.Load(); n != 2 { - t.Fatalf("upstream sent a document %d times, want 2: the abbreviated one was not fetched", n) - } - - store.mu.Lock() - defer store.mu.Unlock() - if got := string(store.files[metadataStoragePath("npm", "big")]); got != full { - t.Errorf("stored packument was replaced: %.80s", got) - } - for path := range store.files { - if strings.Contains(path, ".staged-") { - t.Errorf("staged copy %s left in storage", path) - } - } -} - -// TestNPMStreamedPackumentStorageFails checks that a packument is still -// served, read into memory, when storing it fails. -func TestNPMStreamedPackumentStorageFails(t *testing.T) { - u := newStreamUpstream(t, bigPackument(10)) - _, store, handler := streamProxy(t, u) - store.storeErr = errors.New("no space left on device") - doc := decodeStreamed(t, serve(handler, "/big", nil)) - if len(doc.Versions) != 10 || doc.DistTags["latest"] != "1.0.9" { - t.Errorf("served %d versions, latest %q; want 10 and 1.0.9", len(doc.Versions), doc.DistTags["latest"]) - } -} - -// TestNPMStreamedVersionMetadata resolves versions and dist-tags of a -// packument larger than metadata_max_size. -func TestNPMStreamedVersionMetadata(t *testing.T) { - body := bigPackument(100) - u := newStreamUpstream(t, body) - proxy, _, handler := streamProxy(t, u) - proxy.MetadataMaxSize = int64(len(body) / 4) - - for path, want := range map[string]string{ - "/big/1.0.5": "1.0.5", - "/big/latest": "1.0.99", // 9.0.0 is inside cooldown - "/big/old": "1.0.0", - } { - rec := serve(handler, path, nil) - var v struct { - Version string `json:"version"` - Dist struct { - Tarball string `json:"tarball"` - } `json:"dist"` - } - if rec.Code != http.StatusOK || json.Unmarshal(rec.Body.Bytes(), &v) != nil { - t.Errorf("%s: status %d: %.200s", path, rec.Code, rec.Body.String()) - continue - } - if v.Version != want || v.Dist.Tarball != "http://proxy.example/npm/big/-/big-"+want+".tgz" { - t.Errorf("%s = %s with tarball %s, want %s", path, v.Version, v.Dist.Tarball, want) - } - } - if rec := serve(handler, "/big/9.0.0", nil); rec.Code != http.StatusNotFound { - t.Errorf("version inside cooldown: status %d, want 404", rec.Code) - } - if n := u.requests.Load(); n != 1 { - t.Errorf("upstream asked %d times, want once", n) - } -} - -// TestNPMStreamedTarballCooldown checks the tarball download against a -// packument larger than metadata_max_size: the version's publish time and -// tarball URL are read from storage. -func TestNPMStreamedTarballCooldown(t *testing.T) { - u := newStreamUpstream(t, "") - u.body.Store(bigPackumentAt(u.URL, 100)) - proxy, _, handler := streamProxy(t, u) - proxy.MetadataMaxSize = 1024 - fetcher := proxy.Fetcher.(*mockFetcher) - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} - - if rec := serve(handler, "/big/-/big-9.0.0.tgz", nil); rec.Code != http.StatusNotFound { - t.Errorf("tarball inside cooldown: status %d, want 404", rec.Code) - } - if rec := serve(handler, "/big/-/big-1.0.5.tgz", nil); rec.Code != http.StatusOK || rec.Body.String() != "package" { - t.Errorf("tarball: status %d, body %q", rec.Code, rec.Body.String()) - } - if want := u.URL + "/big/-/big-1.0.5.tgz"; fetcher.fetchedURL != want { - t.Errorf("fetched %s, want %s from the packument", fetcher.fetchedURL, want) - } -} diff --git a/internal/handler/npm_tarball_test.go b/internal/handler/npm_tarball_test.go deleted file mode 100644 index 6bca0f6..0000000 --- a/internal/handler/npm_tarball_test.go +++ /dev/null @@ -1,426 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - registryclient "github.com/git-pkgs/registries/client" - "github.com/git-pkgs/registries/fetch" -) - -func TestNPMContentAddressedTarball(t *testing.T) { - for _, tc := range []struct { - cacheMetadata, stream bool - metadata, tarballs int64 - }{ - {false, false, 2, 1}, - {false, true, 3, 2}, - {true, false, 1, 1}, - {true, true, 1, 2}, - } { - t.Run(fmt.Sprintf("metadata=%t/stream=%t", tc.cacheMetadata, tc.stream), func(t *testing.T) { - var metadataCalls, tarballCalls atomic.Int64 - upstream := npmContentAddressedRegistry(t, &metadataCalls, &tarballCalls) - p, _, _, _ := setupTestProxy(t) - p.CacheMetadata = tc.cacheMetadata - p.MetadataTTL = time.Hour - p.StreamArtifacts = tc.stream - p.Cooldown = &cooldown.Config{Default: "7d"} - p.HTTPClient = upstream.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - p.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - h := NewNPMHandler(p, "http://proxy.test", upstream.URL+"/registry") - routes := http.StripPrefix("/npm", h.Routes()) - metadata := httptest.NewRecorder() - routes.ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/npm/@example/widget", nil)) - if metadata.Code != http.StatusOK { - t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) - } - tarball := npmVersionTarball(metadata.Body.Bytes(), testVersion100) - if want := "http://proxy.test/npm/@example%2Fwidget/-/widget-1.0.0.tgz"; tarball != want { - t.Fatalf("rewritten tarball = %q, want %q", tarball, want) - } - for i := range 2 { - response := httptest.NewRecorder() - routes.ServeHTTP(response, httptest.NewRequest(http.MethodGet, tarball, nil)) - if response.Code != http.StatusOK || response.Body.String() != "tarball bytes" { - t.Fatalf("download %d = %d %q", i, response.Code, response.Body.String()) - } - if !tc.stream { - upstream.Close() - } - } - if metadataCalls.Load() != tc.metadata || tarballCalls.Load() != tc.tarballs { - t.Errorf("upstream requests: metadata=%d tarballs=%d, want %d %d", metadataCalls.Load(), tarballCalls.Load(), tc.metadata, tc.tarballs) - } - }) - } -} - -func npmContentAddressedRegistry(t *testing.T, metadataCalls, tarballCalls *atomic.Int64) *httptest.Server { - t.Helper() - var upstream *httptest.Server - upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/registry/@example/widget": - metadataCalls.Add(1) - w.Header().Set("Content-Type", contentTypeJSON) - _, _ = fmt.Fprintf(w, `{"name":"@example/widget","time":{"1.0.0":%q},"versions":{"1.0.0":{"dist":{"tarball":%q}}}}`, - time.Now().Add(-30*24*time.Hour).Format(time.RFC3339), upstream.URL+"/registry/download/@example/widget/1.0.0/abc123?token=example") - case "/registry/download/@example/widget/1.0.0/abc123": - if r.URL.RawQuery != "token=example" { - t.Errorf("tarball query = %q", r.URL.RawQuery) - } - tarballCalls.Add(1) - _, _ = io.WriteString(w, "tarball bytes") - default: - t.Errorf("unexpected upstream request: %s", r.URL) - http.NotFound(w, r) - } - })) - t.Cleanup(upstream.Close) - return upstream -} - -func TestNPMTarballMetadataFallback(t *testing.T) { - for _, body := range []string{`{"versions":{}}`, `{"versions":{"1.0.0":{"dist":{}}}}`} { - t.Run(body, func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, body) - })) - t.Cleanup(upstream.Close) - p, _, _, fetcher := setupTestProxy(t) - p.CacheMetadata = true - p.MetadataTTL = time.Hour - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} - h := NewNPMHandler(p, "http://proxy.test", upstream.URL) - if _, _, err := p.FetchOrCacheMetadata(t.Context(), "npm", "@example/widget", upstream.URL); err != nil { - t.Fatal(err) - } - upstream.Close() - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/@example/widget/-/widget-1.0.0.tgz", nil)) - if response.Code != http.StatusOK { - t.Fatalf("status = %d: %s", response.Code, response.Body.String()) - } - if want := upstream.URL + "/@example/widget/-/widget-1.0.0.tgz"; fetcher.fetchedURL != want { - t.Errorf("fetched URL = %q, want %q", fetcher.fetchedURL, want) - } - }) - } -} - -// TestNPMTarballWithoutMetadataIsNotFetched checks that a tarball whose -// packument can't be read, so its integrity is unknown, is neither fetched -// nor cached. -func TestNPMTarballWithoutMetadataIsNotFetched(t *testing.T) { - for _, tc := range []struct { - name string - status int - body string - limit int64 - }{ - {name: "unavailable", status: http.StatusServiceUnavailable, body: "unavailable"}, - {name: "over metadata_max_size", status: http.StatusOK, body: `{"versions":{"1.0.0":{"dist":{"integrity":"` + sha512SRI("package") + `"}}}}`, limit: 16}, - } { - t.Run(tc.name, func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(tc.status) - _, _ = io.WriteString(w, tc.body) - })) - t.Cleanup(upstream.Close) - p, _, _, fetcher := setupTestProxy(t) - if tc.limit > 0 { - p.MetadataMaxSize = tc.limit - } - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("tampered"))} - h := NewNPMHandler(p, "http://proxy.test", upstream.URL) - - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/@example/widget/-/widget-1.0.0.tgz", nil)) - if response.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want 502: %s", response.Code, response.Body.String()) - } - if fetcher.fetchedURL != "" { - t.Errorf("fetched %q without the metadata to verify it", fetcher.fetchedURL) - } - cached, err := p.GetCachedArtifact(t.Context(), "npm", "@example/widget", "1.0.0", "widget-1.0.0.tgz") - if err != nil { - t.Fatal(err) - } - if cached != nil { - _ = cached.Reader.Close() - t.Error("tarball was cached without verification") - } - }) - } -} - -// serveEmptyPackuments answers every metadata request with a packument that -// lists no versions, for tests about the tarball download itself. -func serveEmptyPackuments(p *Proxy) { - p.HTTPClient = &http.Client{Transport: pypiRoundTripFunc(func(r *http.Request) (*http.Response, error) { - return pypiHTTPResponse(r, "application/json", `{"versions":{}}`), nil - })} -} - -func TestNPMTarballRejectsUnsafeMetadataURL(t *testing.T) { - for _, target := range []string{ - "https://outside.invalid/package.tgz", "//outside.invalid/package.tgz", "/registry/package.tgz", - "UPSTREAM/outside/package.tgz", "UPSTREAM/registry-other/package.tgz", - "UPSTREAM/registry/../outside/package.tgz", "UPSTREAM/registry/%2e%2e/outside/package.tgz", - "UPSTREAM/registry/%252e%252e/outside/package.tgz", "UPSTREAM/registry/..%2foutside/package.tgz", - "UPSTREAM/registry/..%5coutside/package.tgz", "UPSTREAM/registry/package.tgz#fragment", - "USERINFO/registry/package.tgz", "SCHEME/registry/package.tgz", - } { - t.Run(target, func(t *testing.T) { - var upstream *httptest.Server - upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - address, _ := url.Parse(upstream.URL) - address.User = url.UserPassword("user", "secret") - tarball := strings.NewReplacer("UPSTREAM", upstream.URL, "USERINFO", address.String(), "SCHEME", strings.Replace(upstream.URL, "http:", "https:", 1)).Replace(target) - _ = json.NewEncoder(w).Encode(map[string]any{"versions": map[string]any{testVersion100: map[string]any{"dist": map[string]string{"tarball": tarball}}}}) - })) - t.Cleanup(upstream.Close) - p, _, _, fetcher := setupTestProxy(t) - h := NewNPMHandler(p, "http://proxy.test", upstream.URL+"/registry") - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-1.0.0.tgz", nil)) - if response.Code != http.StatusBadGateway || fetcher.fetchCalled { - t.Errorf("status = %d, fetched = %t; body: %s", response.Code, fetcher.fetchCalled, response.Body.String()) - } - }) - } -} - -// TestNPMTarballCooldownRefetchesStalePackument checks that a tarball for a -// version the cached packument does not list yet is decided on the packument -// upstream serves now, not let through for want of a publish time. -func TestNPMTarballCooldownRefetchesStalePackument(t *testing.T) { - for _, tc := range []struct { - name string - published time.Time - down bool - want int - }{ - {"recent", time.Now().Add(-time.Hour), false, http.StatusNotFound}, - // Without upstream the version can't be checked, so it isn't served. - {"upstream down", time.Now().Add(-time.Hour), true, http.StatusBadGateway}, - // The refetched entry's integrity is checked, so a tarball that - // doesn't match it is refused rather than served unverified. - {"old", time.Now().Add(-30 * 24 * time.Hour), false, http.StatusBadGateway}, - } { - t.Run(tc.name, func(t *testing.T) { - var released atomic.Bool - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - old := time.Now().Add(-60 * 24 * time.Hour).Format(time.RFC3339) - if released.Load() && tc.down { - w.WriteHeader(http.StatusServiceUnavailable) - return - } - if !released.Load() { - _, _ = fmt.Fprintf(w, `{"name":"widget","time":{"1.0.0":%q},"versions":{"1.0.0":{"dist":{}}}}`, old) - return - } - _, _ = fmt.Fprintf(w, `{"name":"widget","time":{"1.0.0":%q,"2.0.0":%q},"versions":{"1.0.0":{"dist":{}},"2.0.0":{"dist":{"integrity":%q}}}}`, - old, tc.published.Format(time.RFC3339), sha512SRI("package")) - })) - t.Cleanup(upstream.Close) - p, _, _, fetcher := setupTestProxy(t) - p.CacheMetadata = true - p.MetadataTTL = time.Hour - p.Cooldown = &cooldown.Config{Default: "7d"} - p.HTTPClient = upstream.Client() - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("tampered"))} - h := NewNPMHandler(p, "http://proxy.test", upstream.URL) - - metadata := httptest.NewRecorder() - h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) - if metadata.Code != http.StatusOK { - t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) - } - released.Store(true) - - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) - if response.Code != tc.want { - t.Fatalf("status = %d, want %d: %s", response.Code, tc.want, response.Body.String()) - } - }) - } -} - -// widgetPackument is a packument listing only widget 1.0.0, published long -// ago. -func widgetPackument() string { - old := time.Now().Add(-60 * 24 * time.Hour).Format(time.RFC3339) - return fmt.Sprintf(`{"name":"widget","time":{"1.0.0":%q},"versions":{"1.0.0":{"dist":{}}}}`, old) -} - -// cooldownNPMHandler returns an npm handler for upstream with cooldown on -// and a tarball fetcher that would serve anything asked of it. -func cooldownNPMHandler(t *testing.T, upstream *httptest.Server, ttl time.Duration) *NPMHandler { - t.Helper() - p, _, _, fetcher := setupTestProxy(t) - p.CacheMetadata = true - p.MetadataTTL = ttl - p.Cooldown = &cooldown.Config{Default: "7d"} - p.HTTPClient = upstream.Client() - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} - return NewNPMHandler(p, "http://proxy.test", upstream.URL) -} - -// TestNPMTarballCooldownStalePackument checks that a stale packument served -// because upstream failed doesn't let through a version it doesn't list, -// and that the upstream failure decides the response. -func TestNPMTarballCooldownStalePackument(t *testing.T) { - for _, tc := range []struct { - name string - status int - limited bool - want int - }{ - {"upstream down", http.StatusServiceUnavailable, false, http.StatusBadGateway}, - {"not found", http.StatusNotFound, false, http.StatusNotFound}, - {"rate limited", http.StatusTooManyRequests, true, http.StatusServiceUnavailable}, - } { - t.Run(tc.name, func(t *testing.T) { - var down atomic.Bool - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if down.Load() { - w.Header().Set("Retry-After", "60") - w.WriteHeader(tc.status) - return - } - _, _ = io.WriteString(w, widgetPackument()) - })) - t.Cleanup(upstream.Close) - // With no TTL the packument is read in memory and every read - // asks upstream. - h := cooldownNPMHandler(t, upstream, 0) - if tc.limited { - gate := registryclient.NewRateLimitGate(http.DefaultTransport) - h.proxy.HTTPClient = &http.Client{Transport: gate} - h.proxy.RateLimits = gate - } - - metadata := httptest.NewRecorder() - h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) - if metadata.Code != http.StatusOK { - t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) - } - down.Store(true) - - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) - if response.Code != tc.want { - t.Fatalf("status = %d, want %d: %s", response.Code, tc.want, response.Body.String()) - } - if tc.limited && response.Header().Get("Retry-After") == "" { - t.Error("no Retry-After on a rate-limited response") - } - }) - } -} - -// TestNPMTarballCooldownFetchesMissingVersionOnce checks that a packument -// just fetched from upstream isn't fetched again because it doesn't list the -// requested version. -func TestNPMTarballCooldownFetchesMissingVersionOnce(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - requests.Add(1) - _, _ = io.WriteString(w, widgetPackument()) - })) - t.Cleanup(upstream.Close) - h := cooldownNPMHandler(t, upstream, time.Hour) - - response := httptest.NewRecorder() - h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-99.0.0.tgz", nil)) - if n := requests.Load(); n != 1 { - t.Errorf("upstream packument requests = %d, want 1", n) - } -} - -// TestNPMTarballCooldownRefreshIgnoresSharedStale checks that a refetch of a -// stored packument lacking the version, joining another caller's fetch that -// falls back to the stored copy, doesn't take that copy as upstream's answer. -func TestNPMTarballCooldownRefreshIgnoresSharedStale(t *testing.T) { - var blocked atomic.Bool - arrived := make(chan struct{}, 1) - release := make(chan struct{}) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !blocked.Load() { - _, _ = io.WriteString(w, widgetPackument()) - return - } - arrived <- struct{}{} - select { - case <-release: - case <-r.Context().Done(): - } - w.WriteHeader(http.StatusServiceUnavailable) - })) - t.Cleanup(upstream.Close) - h := cooldownNPMHandler(t, upstream, time.Hour) - p := h.proxy - - metadata := httptest.NewRecorder() - h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) - if metadata.Code != http.StatusOK { - t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) - } - registryURL, cacheKey := h.sourceFor("widget") - packumentURL := registryURL + "/widget" - accept := h.packumentAccept() - row, err := p.DB.GetMetadataCache("npm", cacheKey) - if err != nil || row == nil { - t.Fatalf("no stored packument row: %v", err) - } - setFetchedAt := func(at time.Time) { - t.Helper() - if ok, err := p.DB.RefreshMetadataCache("npm", cacheKey, row.ContentDigest.String, at); err != nil || !ok { - t.Fatalf("setting fetched_at: %t, %v", ok, err) - } - } - - // Another caller revalidates the expired copy, and upstream holds it. - setFetchedAt(time.Now().Add(-2 * time.Hour)) - blocked.Store(true) - other := make(chan error, 1) - go func() { - _, err := p.storedMetadataDocument(context.Background(), "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator()) - other <- err - }() - <-arrived - - // The tarball request reads the copy as within its TTL and refetches it, - // joining that fetch. - setFetchedAt(time.Now()) - response := httptest.NewRecorder() - done := make(chan struct{}) - go func() { - defer close(done) - h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) - }() - waitForMetadataWaiters(t, p, metadataCoalesceKey("npm", cacheKey, packumentURL, accept, "", false)+"\x00stored", 1) - close(release) - <-done - if err := <-other; err != nil { - t.Fatalf("other caller: %v", err) - } - if response.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusBadGateway, response.Body.String()) - } -} diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index 9b9a6c3..bc1edde 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -1,23 +1,14 @@ package handler import ( - "bytes" - "compress/gzip" "encoding/json" - "errors" - "fmt" - "io" "log/slog" "net/http" "net/http/httptest" - "strings" - "sync/atomic" "testing" - "testing/iotest" "time" "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/registries/fetch" ) const testVersion100 = "1.0.0" @@ -55,86 +46,6 @@ func TestNPMExtractVersionFromFilename(t *testing.T) { } } -func TestNPMHandlerUsesConfiguredUpstream(t *testing.T) { - t.Run("metadata", func(t *testing.T) { - var requestPath, authHeader string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requestPath = r.URL.Path - authHeader = r.Header.Get("Authorization") - if authHeader != "Bearer npm-token" { - w.WriteHeader(http.StatusUnauthorized) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"versions":{}}`) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - authorizeUpstream(proxy, "Bearer npm-token") - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL+"/root/") - - req := httptest.NewRequest(http.MethodGet, "/testpkg", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if requestPath != "/root/testpkg" { - t.Errorf("upstream path = %q, want %q", requestPath, "/root/testpkg") - } - if authHeader != "Bearer npm-token" { - t.Errorf("Authorization = %q, want %q", authHeader, "Bearer npm-token") - } - }) - - t.Run("download", func(t *testing.T) { - proxy, _, _, artifactFetcher := setupTestProxy(t) - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("package")), - ContentType: "application/gzip", - } - serveEmptyPackuments(proxy) - h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/") - - req := httptest.NewRequest(http.MethodGet, "/testpkg/-/testpkg-1.0.0.tgz", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - want := "https://npm.example.test/root/testpkg/-/testpkg-1.0.0.tgz" - if artifactFetcher.fetchedURL != want { - t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) - } - }) - - t.Run("scoped download", func(t *testing.T) { - proxy, _, _, artifactFetcher := setupTestProxy(t) - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("package")), - ContentType: "application/gzip", - } - serveEmptyPackuments(proxy) - h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/") - - req := httptest.NewRequest(http.MethodGet, "/@scope/name/-/name-1.0.0.tgz", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - want := "https://npm.example.test/root/@scope/name/-/name-1.0.0.tgz" - if artifactFetcher.fetchedURL != want { - t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) - } - }) -} - func TestNPMRewriteMetadata(t *testing.T) { h := &NPMHandler{ proxy: testProxy(), @@ -155,7 +66,7 @@ func TestNPMRewriteMetadata(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping("lodash", []byte(input)) + output, err := h.rewriteMetadata("lodash", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -176,26 +87,6 @@ func TestNPMRewriteMetadata(t *testing.T) { } } -func TestNPMRewriteMetadataLogsOncePerDocument(t *testing.T) { - var logs bytes.Buffer - proxy := testProxy() - proxy.Logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) - h := &NPMHandler{proxy: proxy, proxyURL: "http://localhost:8080"} - - input := `{"name":"lodash","versions":{ - "4.17.19":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz"}}, - "4.17.20":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz"}}, - "4.17.21":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"}}}}` - if _, _, err := h.rewriteMetadataKeeping("lodash", []byte(input)); err != nil { - t.Fatalf("rewriteMetadata failed: %v", err) - } - - lines := strings.Split(strings.TrimSpace(logs.String()), "\n") - if len(lines) != 1 || !strings.Contains(lines[0], `msg="rewrote tarball URLs"`) || !strings.Contains(lines[0], "count=3") { - t.Errorf("logs = %q, want one line counting 3 rewrites", logs.String()) - } -} - func TestNPMRewriteMetadataScopedPackage(t *testing.T) { h := &NPMHandler{ proxy: testProxy(), @@ -215,7 +106,7 @@ func TestNPMRewriteMetadataScopedPackage(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping("@babel/core", []byte(input)) + output, err := h.rewriteMetadata("@babel/core", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -330,7 +221,7 @@ func TestNPMRewriteMetadataCooldown(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input)) + output, err := h.rewriteMetadata("testpkg", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -386,7 +277,7 @@ func TestNPMRewriteMetadataCooldownExemptPackage(t *testing.T) { } }` - output, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input)) + output, err := h.rewriteMetadata("testpkg", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -422,7 +313,7 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) { })) defer upstream.Close() - t.Run("no cooldown uses combined accept header", func(t *testing.T) { + t.Run("no cooldown uses abbreviated metadata", func(t *testing.T) { h := &NPMHandler{ proxy: testProxy(), upstreamURL: upstream.URL, @@ -433,12 +324,12 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) { w := httptest.NewRecorder() h.handlePackageMetadata(w, req) - if gotAccept != npmAcceptDefault { - t.Errorf("Accept = %q, want %q", gotAccept, npmAcceptDefault) + if gotAccept != npmAbbreviatedCT { + t.Errorf("Accept = %q, want abbreviated metadata header", gotAccept) } }) - t.Run("cooldown enabled uses full metadata only", func(t *testing.T) { + t.Run("cooldown enabled uses full metadata", func(t *testing.T) { proxy := testProxy() proxy.Cooldown = &cooldown.Config{Default: "3d"} @@ -452,27 +343,8 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) { w := httptest.NewRecorder() h.handlePackageMetadata(w, req) - if gotAccept != contentTypeJSON { - t.Errorf("Accept = %q, want %q (cooldown requires full metadata)", gotAccept, contentTypeJSON) - } - }) - - t.Run("full metadata option uses full metadata without cooldown", func(t *testing.T) { - proxy := testProxy() - proxy.NPMFullMetadata = true - - h := &NPMHandler{ - proxy: proxy, - upstreamURL: upstream.URL, - proxyURL: "http://proxy.local", - } - - req := httptest.NewRequest(http.MethodGet, "/testpkg", nil) - w := httptest.NewRecorder() - h.handlePackageMetadata(w, req) - - if gotAccept != contentTypeJSON { - t.Errorf("Accept = %q, want %q (npm_full_metadata requires full metadata)", gotAccept, contentTypeJSON) + if gotAccept == npmAbbreviatedCT { + t.Error("cooldown enabled should use full metadata, not abbreviated") } }) } @@ -499,933 +371,3 @@ func TestNPMHandlerMetadataNotFound(t *testing.T) { t.Errorf("status = %d, want %d", w.Code, http.StatusNotFound) } } - -func TestNPMDownloadCooldown(t *testing.T) { - now := time.Now() - packument := `{ - "name": "leftpad", - "dist-tags": {"latest": "2.0.0"}, - "time": { - "1.0.0": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `", - "2.0.0": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `" - }, - "versions": {"1.0.0": {}, "2.0.0": {}} - }` - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", contentTypeJSON) - _, _ = io.WriteString(w, packument) - })) - defer upstream.Close() - - tests := []struct { - name string - version string - wantStatus int - }{ - {"published before the window serves the tarball", testVersion100, http.StatusOK}, - {"published inside the window is withheld", "2.0.0", http.StatusNotFound}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "7d"} - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("tarball data")), - ContentType: "application/octet-stream", - } - - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-" + tt.version + ".tgz") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != tt.wantStatus { - t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus) - } - if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled { - t.Error("fetched a version that is still inside the cooldown window") - } - }) - } -} - -func TestNPMDownloadCooldownDisabled(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - h := NewNPMHandler(proxy, "http://proxy.test", "") - - if inCooldown, _ := h.versionInCooldown("leftpad", testVersion100, func() (npmVersionData, error) { - t.Fatal("cooldown must not request metadata when disabled") - return npmVersionData{}, nil - }); inCooldown { - t.Error("versionInCooldown = true, want false when cooldown is not configured") - } -} - -func TestNPMDownloadCooldownUsesStoredPublishTime(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - t.Error("metadata must not be fetched when the publish time is already stored") - w.WriteHeader(http.StatusInternalServerError) - })) - defer upstream.Close() - - tests := []struct { - name string - version string - publishedAt time.Time - wantStatus int - }{ - {"stored time before the window serves the tarball", testVersion100, time.Now().Add(-30 * 24 * time.Hour), http.StatusOK}, - {"stored time inside the window is withheld", "2.0.0", time.Now().Add(-1 * time.Hour), http.StatusNotFound}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, db, store, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "7d"} - seedPackage(t, db, store, "npm", "leftpad", tt.version, "leftpad-"+tt.version+".tgz", "tarball data") - - if err := db.SetVersionPublishedAt("pkg:npm/leftpad@"+tt.version, "pkg:npm/leftpad", tt.publishedAt); err != nil { - t.Fatalf("seeding publish time failed: %v", err) - } - - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-" + tt.version + ".tgz") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != tt.wantStatus { - t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus) - } - }) - } -} - -func TestNPMDownloadCooldownFetchesMetadataOnce(t *testing.T) { - now := time.Now() - packument := `{ - "name": "leftpad", - "dist-tags": {"latest": "1.0.0"}, - "time": { - "1.0.0": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `" - }, - "versions": {"1.0.0": {}} - }` - - var metadataRequests atomic.Int64 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - metadataRequests.Add(1) - w.Header().Set("Content-Type", contentTypeJSON) - _, _ = io.WriteString(w, packument) - })) - defer upstream.Close() - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "7d"} - - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - // The first download parses the packument once and persists the publish - // time; caching the artifact afterwards upserts the versions row without a - // publish time, which must not erase the stored value. The second download - // must answer from the stored time alone. - for i := 0; i < 2; i++ { - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("tarball data")), - ContentType: "application/octet-stream", - } - resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-" + testVersion100 + ".tgz") - if err != nil { - t.Fatalf("request %d failed: %v", i+1, err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("request %d status = %d, want %d", i+1, resp.StatusCode, http.StatusOK) - } - } - - if got := metadataRequests.Load(); got != 1 { - t.Errorf("metadata requests = %d, want 1", got) - } -} - -// TestNPMDownloadErrorResponsesAreJSON guards against a regression where -// routing handleDownload's error path through the shared serveArtifactError -// helper silently switched npm's 404/502 tarball error bodies from JSON to -// plain text; npm clients expect a JSON {"error": "..."} body on every -// download failure, including the newer scan-blocked (403) case. -func TestNPMDownloadErrorResponsesAreJSON(t *testing.T) { - tests := []struct { - name string - fetchErr error - blocked bool - wantStatus int - }{ - {"upstream not found", fetch.ErrNotFound, false, http.StatusNotFound}, - {"upstream failure", errors.New("connection refused"), false, http.StatusBadGateway}, - {"blocked by scan", nil, true, http.StatusForbidden}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - if tt.blocked { - proxy.Scanners = newTestScanGroup(t, newTestScanServer(t, false, "malware detected").URL, false) - } - fetcher.fetchErr = tt.fetchErr - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("tarball data")), - ContentType: "application/octet-stream", - } - - serveEmptyPackuments(proxy) - h := NewNPMHandler(proxy, "http://proxy.test", "http://upstream.invalid") - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-1.0.0.tgz") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != tt.wantStatus { - t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus) - } - if ct := resp.Header.Get("Content-Type"); ct != contentTypeJSON { - t.Errorf("Content-Type = %q, want %q", ct, contentTypeJSON) - } - var body map[string]any - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("response body is not valid JSON: %v", err) - } - if _, ok := body["error"]; !ok { - t.Errorf("response body %v missing \"error\" key", body) - } - }) - } -} - -// newNPMAuditUpstream returns a handler pointed at a stub registry, plus the -// last request that registry saw. -func newNPMAuditUpstream(t *testing.T, respond http.HandlerFunc) (*NPMHandler, *npmAuditCapture) { - t.Helper() - - capture := &npmAuditCapture{} - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - capture.method = r.Method - capture.path = r.URL.Path - capture.query = r.URL.RawQuery - capture.contentType = r.Header.Get("Content-Type") - capture.contentEncoding = r.Header.Get("Content-Encoding") - capture.authorization = r.Header.Get("Authorization") - capture.body, _ = io.ReadAll(r.Body) - respond(w, r) - })) - t.Cleanup(upstream.Close) - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - return NewNPMHandler(proxy, "http://proxy.test", upstream.URL), capture -} - -type npmAuditCapture struct { - method string - path string - query string - contentType string - contentEncoding string - authorization string - body []byte -} - -func npmAuditJSON(body string) http.HandlerFunc { - return func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, body) - } -} - -func serveNPM(t *testing.T, h *NPMHandler, method, target string, body io.Reader, - headers map[string]string, -) *httptest.ResponseRecorder { - t.Helper() - - req := httptest.NewRequest(method, target, body) - for name, value := range headers { - req.Header.Set(name, value) - } - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - return w -} - -func TestNPMAuditRelaysRequestAndResponse(t *testing.T) { - const report = `{"actions":[],"advisories":{},"metadata":{"vulnerabilities":{"total":0}}}` - const payload = `{"name":"app","requires":{"lodash":"^4.17.21"},"dependencies":{}}` - - h, got := newNPMAuditUpstream(t, npmAuditJSON(report)) - w := serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits?foo=bar", - strings.NewReader(payload), map[string]string{"Content-Type": "application/json"}) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if w.Body.String() != report { - t.Errorf("body = %q, want %q", w.Body.String(), report) - } - if ct := w.Header().Get("Content-Type"); ct != "application/json" { - t.Errorf("response Content-Type = %q, want application/json", ct) - } - if got.method != http.MethodPost { - t.Errorf("upstream method = %q, want POST", got.method) - } - if got.path != "/-/npm/v1/security/audits" { - t.Errorf("upstream path = %q, want /-/npm/v1/security/audits", got.path) - } - if got.query != "foo=bar" { - t.Errorf("upstream query = %q, want foo=bar", got.query) - } - if string(got.body) != payload { - t.Errorf("upstream body = %q, want %q", got.body, payload) - } - if got.contentType != "application/json" { - t.Errorf("upstream Content-Type = %q, want application/json", got.contentType) - } -} - -// npm gzips its audit payload, so the bytes and the header describing them -// must travel together. -func TestNPMAuditForwardsGzippedBody(t *testing.T) { - var gzipped bytes.Buffer - zw := gzip.NewWriter(&gzipped) - if _, err := io.WriteString(zw, `{"name":"app"}`); err != nil { - t.Fatalf("writing gzip body: %v", err) - } - if err := zw.Close(); err != nil { - t.Fatalf("closing gzip writer: %v", err) - } - want := gzipped.Bytes() - - h, got := newNPMAuditUpstream(t, npmAuditJSON(`{}`)) - w := serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits", - bytes.NewReader(want), map[string]string{ - "Content-Type": "application/json", - "Content-Encoding": "gzip", - }) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if got.contentEncoding != "gzip" { - t.Errorf("upstream Content-Encoding = %q, want gzip", got.contentEncoding) - } - if !bytes.Equal(got.body, want) { - t.Errorf("upstream body was altered: got %d bytes, want %d", len(got.body), len(want)) - } -} - -func TestNPMAuditCoversAllSecurityEndpoints(t *testing.T) { - paths := []string{ - "/-/npm/v1/security/audits", // pnpm audit, npm audit (full) - "/-/npm/v1/security/audits/quick", // yarn npm audit, npm audit fallback - "/-/npm/v1/security/advisories/bulk", // npm audit (npm 7+) - } - - for _, path := range paths { - t.Run(path, func(t *testing.T) { - h, got := newNPMAuditUpstream(t, npmAuditJSON(`{}`)) - w := serveNPM(t, h, http.MethodPost, path, strings.NewReader(`{}`), nil) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if got.path != path { - t.Errorf("upstream path = %q, want %q", got.path, path) - } - }) - } -} - -func TestNPMAuditAppliesUpstreamAuth(t *testing.T) { - h, got := newNPMAuditUpstream(t, npmAuditJSON(`{}`)) - authorizeUpstream(h.proxy, "Bearer npm-token") - - serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits", strings.NewReader(`{}`), nil) - - if got.authorization != "Bearer npm-token" { - t.Errorf("Authorization = %q, want %q", got.authorization, "Bearer npm-token") - } -} - -func TestNPMAuditRelaysUpstreamError(t *testing.T) { - const upstreamBody = `{"error":"unauthorized"}` - - h, _ := newNPMAuditUpstream(t, func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusUnauthorized) - _, _ = io.WriteString(w, upstreamBody) - }) - w := serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits", strings.NewReader(`{}`), nil) - - if w.Code != http.StatusUnauthorized { - t.Errorf("status = %d, want %d", w.Code, http.StatusUnauthorized) - } - if w.Body.String() != upstreamBody { - t.Errorf("body = %q, want %q", w.Body.String(), upstreamBody) - } -} - -// A proxy-side failure must still be JSON, or the client reports it as a -// malformed audit response. -func TestNPMAuditUpstreamUnreachable(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) - upstreamURL := upstream.URL - upstream.Close() // nothing is listening now - - proxy, _, _, _ := setupTestProxy(t) - h := NewNPMHandler(proxy, "http://proxy.test", upstreamURL) - - w := serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits", strings.NewReader(`{}`), nil) - - if w.Code != http.StatusBadGateway { - t.Errorf("status = %d, want %d", w.Code, http.StatusBadGateway) - } - if ct := w.Header().Get("Content-Type"); ct != "application/json" { - t.Errorf("Content-Type = %q, want application/json", ct) - } - if !json.Valid(w.Body.Bytes()) { - t.Errorf("body is not valid JSON: %q", w.Body.String()) - } -} - -func TestNPMAuditRejectsBadRequests(t *testing.T) { - t.Run("non-POST method", func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test") - - w := serveNPM(t, h, http.MethodGet, "/-/npm/v1/security/audits", nil, nil) - - if w.Code != http.StatusMethodNotAllowed { - t.Errorf("status = %d, want %d", w.Code, http.StatusMethodNotAllowed) - } - if !json.Valid(w.Body.Bytes()) { - t.Errorf("body is not valid JSON: %q", w.Body.String()) - } - }) - - t.Run("body over the size cap", func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test") - - body := strings.NewReader(strings.Repeat("a", npmSecurityMaxBody+1)) - w := serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits", body, nil) - - if w.Code != http.StatusRequestEntityTooLarge { - t.Errorf("status = %d, want %d", w.Code, http.StatusRequestEntityTooLarge) - } - }) - - t.Run("unreadable body is not reported as too large", func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test") - - req := httptest.NewRequest(http.MethodPost, "/-/npm/v1/security/audits", - iotest.TimeoutReader(strings.NewReader("{}"))) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusBadRequest { - t.Errorf("status = %d, want %d", w.Code, http.StatusBadRequest) - } - if !json.Valid(w.Body.Bytes()) { - t.Errorf("body is not valid JSON: %q", w.Body.String()) - } - }) -} - -func TestNPMAuditDoesNotInjectUpstreamQuery(t *testing.T) { - h, got := newNPMAuditUpstream(t, npmAuditJSON(`{}`)) - - serveNPM(t, h, http.MethodPost, - "/-/npm/v1/security/audits%3Fevil=1", strings.NewReader(`{}`), nil) - - if got.query != "" { - t.Errorf("upstream query = %q, want empty: encoded ? leaked into the query", got.query) - } - if got.path != "/-/npm/v1/security/audits?evil=1" { - t.Errorf("upstream path = %q, want the encoded ? kept in the path", got.path) - } -} - -// The audit POST must go through relayResponse, not copy upstream headers -// wholesale: a relayed Connection header would be honoured downstream. -// TestRelayRoutes covers the GET paths; this covers the POST. -func TestNPMAuditStripsHopByHopHeaders(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - conn, rw, err := w.(http.Hijacker).Hijack() - if err != nil { - t.Error(err) - return - } - defer func() { _ = conn.Close() }() - _, _ = fmt.Fprint(rw, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n"+ - "Connection: X-Private\r\nX-Private: secret\r\nContent-Length: 2\r\n\r\n{}") - _ = rw.Flush() - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - downstream := httptest.NewServer(NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) - defer downstream.Close() - - resp, err := downstream.Client().Post( - downstream.URL+"/-/npm/v1/security/audits", contentTypeJSON, strings.NewReader(`{}`)) - if err != nil { - t.Fatal(err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.Header.Get("Connection") != "" || resp.Header.Get("X-Private") != "" { - t.Errorf("connection-scoped headers leaked: %v", resp.Header) - } -} - -// `npm audit signatures` reads the registry signing keys. The path used to fall -// through to the package dispatch and be escaped into a package name. -func TestNPMKeysProxiesUpstream(t *testing.T) { - const keys = `{"keys":[{"keyid":"SHA256:jl3bwswu","keytype":"ecdsa-sha2-nistp256"}]}` - - var gotPath, gotAuth string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // EscapedPath, not Path: the bug escaped the slashes into a package - // name, and the server decodes %2F back into Path either way. - gotPath, gotAuth = r.URL.EscapedPath(), r.Header.Get("Authorization") - if gotPath != npmKeysPath { - w.WriteHeader(http.StatusMethodNotAllowed) // what registry.npmjs.org answers - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, keys) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - authorizeUpstream(proxy, "Bearer npm-token") - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - - w := serveNPM(t, h, http.MethodGet, npmKeysPath, nil, nil) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } - if w.Body.String() != keys { - t.Errorf("body = %q, want %q", w.Body.String(), keys) - } - if gotPath != npmKeysPath { - t.Errorf("upstream path = %q, want %q", gotPath, npmKeysPath) - } - if gotAuth != "Bearer npm-token" { - t.Errorf("Authorization = %q, want it applied", gotAuth) - } -} - -// Tarball paths share the /-/ prefix with the /-/npm/v1 endpoints. -func TestNPMTarballStillRoutesToDownload(t *testing.T) { - proxy, _, _, artifactFetcher := setupTestProxy(t) - artifactFetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("package")), - ContentType: "application/gzip", - } - serveEmptyPackuments(proxy) - h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test") - - w := serveNPM(t, h, http.MethodGet, "/lodash/-/lodash-4.17.21.tgz", nil, nil) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) - } -} - -func TestNPMMetadataPath(t *testing.T) { - tests := []struct { - path, wantName, wantRegistry string - }{ - {"", "", ""}, - {"lodash", "lodash", "lodash"}, - {"lodash/4.17.21", "lodash", "lodash/4.17.21"}, - {"lodash/latest", "lodash", "lodash/latest"}, - {"@babel/core", "@babel/core", "@babel%2Fcore"}, - {"@babel/core/7.23.0", "@babel/core", "@babel%2Fcore/7.23.0"}, - } - for _, tt := range tests { - name, registry := npmMetadataPath(tt.path) - if name != tt.wantName || registry != tt.wantRegistry { - t.Errorf("npmMetadataPath(%q) = (%q, %q), want (%q, %q)", - tt.path, name, registry, tt.wantName, tt.wantRegistry) - } - } -} - -func TestNPMVersionMetadataKeepsVersionSegment(t *testing.T) { - tests := []struct { - request, upstreamPath, wantTarball string - body string - }{ - { - request: "/lodash/4.17.21", - upstreamPath: "/lodash/4.17.21", - wantTarball: "http://proxy.local/npm/lodash/-/lodash-4.17.21.tgz", - body: `{ - "name": "lodash", - "version": "4.17.21", - "dist": {"tarball": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"} - }`, - }, - { - request: "/@babel/core/7.23.0", - upstreamPath: "/@babel%2Fcore/7.23.0", - wantTarball: "http://proxy.local/npm/@babel%2Fcore/-/core-7.23.0.tgz", - body: `{ - "name": "@babel/core", - "version": "7.23.0", - "dist": {"tarball": "https://registry.npmjs.org/@babel/core/-/core-7.23.0.tgz"} - }`, - }, - } - for _, tt := range tests { - t.Run(tt.request, func(t *testing.T) { - var gotPath string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotPath = r.URL.EscapedPath() - if gotPath != tt.upstreamPath { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, tt.body) - })) - defer upstream.Close() - - h := &NPMHandler{proxy: testProxy(), upstreamURL: upstream.URL, proxyURL: "http://proxy.local"} - w := httptest.NewRecorder() - h.handlePackageMetadata(w, httptest.NewRequest(http.MethodGet, tt.request, nil)) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) - } - if gotPath != tt.upstreamPath { - t.Fatalf("upstream path = %q, want %q", gotPath, tt.upstreamPath) - } - var result map[string]any - if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { - t.Fatalf("parse response: %v", err) - } - if result["dist"].(map[string]any)["tarball"] != tt.wantTarball { - t.Errorf("tarball = %v", result["dist"].(map[string]any)["tarball"]) - } - }) - } -} - -func TestNPMMetadataRejectsEmptyPath(t *testing.T) { - h := NewNPMHandler(testProxy(), "http://proxy.local", "http://unused.test") - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/", nil)) - if w.Code != http.StatusBadRequest { - t.Fatalf("status = %d, want 400; body: %s", w.Code, w.Body.String()) - } -} - -func TestNPMVersionMetadataRejectsTraversal(t *testing.T) { - called := false - upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - called = true - })) - defer upstream.Close() - - h := &NPMHandler{proxy: testProxy(), upstreamURL: upstream.URL, proxyURL: "http://proxy.local"} - req := httptest.NewRequest(http.MethodGet, "/lodash/x", nil) - req.URL.Path = "/lodash/.." - w := httptest.NewRecorder() - h.handlePackageMetadata(w, req) - - if w.Code != http.StatusBadRequest { - t.Fatalf("status = %d, want 400; body: %s", w.Code, w.Body.String()) - } - if called { - t.Fatal("upstream contacted for traversal path") - } -} - -func TestNPMVersionMetadataCooldown(t *testing.T) { - proxyURL, upstreamPaths := npmCooldownMetadataHarness(t) - - resp, err := http.Get(proxyURL + "/demo") - if err != nil { - t.Fatalf("packument request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("packument status = %d, want 200", resp.StatusCode) - } - var packument map[string]any - if err := json.NewDecoder(resp.Body).Decode(&packument); err != nil { - t.Fatalf("decode packument: %v", err) - } - distTags := packument["dist-tags"].(map[string]any) - if distTags["latest"] != testVersion100 { - t.Fatalf("dist-tags.latest = %v, want %s", distTags["latest"], testVersion100) - } - if _, ok := packument["versions"].(map[string]any)["2.0.0"]; ok { - t.Fatal("packument should not include version 2.0.0") - } - - for _, tt := range []struct { - path string - wantStatus int - wantVersion string - wantError string - }{ - {"/demo/latest", http.StatusOK, testVersion100, ""}, - {"/demo/1.0.0", http.StatusOK, testVersion100, ""}, - {"/demo/2.0.0", http.StatusNotFound, "", "version not found"}, - } { - t.Run(tt.path, func(t *testing.T) { - testNPMVersionMetadataCooldownPath(t, proxyURL, upstreamPaths, tt.path, tt.wantStatus, tt.wantVersion, tt.wantError) - }) - } -} - -// testNPMVersionMetadataCooldownPath is a subtest helper so the parent test stays -// under gocognit's complexity limit. -func testNPMVersionMetadataCooldownPath(t *testing.T, proxyURL string, upstreamPaths *[]string, path string, wantStatus int, wantVersion, wantError string) { - t.Helper() - before := len(*upstreamPaths) - resp, err := http.Get(proxyURL + path) - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - body := readNPMTestBody(t, resp.Body) - if resp.StatusCode != wantStatus { - t.Fatalf("status = %d, want %d; body: %s", resp.StatusCode, wantStatus, body) - } - for _, upstreamPath := range (*upstreamPaths)[before:] { - if upstreamPath == path { - t.Fatalf("upstream must not be called for version path %s when answering from the filtered packument", path) - } - } - if wantError != "" { - if !strings.Contains(body, wantError) { - t.Fatalf("error body should mention %q, got %s", wantError, body) - } - return - } - var versionDoc map[string]any - if err := json.Unmarshal([]byte(body), &versionDoc); err != nil { - t.Fatalf("decode response: %v", err) - } - if versionDoc["version"] != wantVersion { - t.Fatalf("version = %v, want %s", versionDoc["version"], wantVersion) - } - wantTarball := "http://proxy.test/npm/demo/-/demo-" + wantVersion + ".tgz" - if versionDoc["dist"].(map[string]any)["tarball"] != wantTarball { - t.Fatalf("tarball = %v, want %s", versionDoc["dist"].(map[string]any)["tarball"], wantTarball) - } -} - -func npmCooldownMetadataHarness(t *testing.T) (proxyURL string, upstreamPaths *[]string) { - t.Helper() - now := time.Now() - old := now.Add(-30 * 24 * time.Hour).Format(time.RFC3339) - recent := now.Add(-1 * time.Hour).Format(time.RFC3339) - packument := `{ - "name": "demo", - "dist-tags": {"latest": "2.0.0"}, - "time": {"1.0.0": "` + old + `", "2.0.0": "` + recent + `"}, - "versions": { - "1.0.0": { - "name": "demo", - "version": "1.0.0", - "dist": {"tarball": "https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"} - }, - "2.0.0": { - "name": "demo", - "version": "2.0.0", - "dist": {"tarball": "https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"} - } - } - }` - version200 := `{"name":"demo","version":"2.0.0","dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}` - version100 := `{"name":"demo","version":"1.0.0","dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"}}` - - var paths []string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - paths = append(paths, r.URL.Path) - w.Header().Set("Content-Type", contentTypeJSON) - switch r.URL.Path { - case "/demo": - _, _ = io.WriteString(w, packument) - case "/demo/latest", "/demo/2.0.0": - _, _ = io.WriteString(w, version200) - case "/demo/1.0.0": - _, _ = io.WriteString(w, version100) - default: - http.NotFound(w, r) - } - })) - t.Cleanup(upstream.Close) - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "7d"} - srv := httptest.NewServer(NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) - t.Cleanup(srv.Close) - return srv.URL, &paths -} - -func readNPMTestBody(t *testing.T, r io.Reader) string { - b, err := io.ReadAll(r) - if err != nil { - t.Fatalf("read body: %v", err) - } - return string(b) -} - -func TestNPMVersionMetadataDenied(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, `{ - "name": "lodash", - "version": "4.17.21", - "dist": {"tarball": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"} - }`) - })) - defer upstream.Close() - - proxy := testProxy() - setTestDenylist(t, proxy, "pkg:npm/lodash@4.17.21") - h := &NPMHandler{proxy: proxy, upstreamURL: upstream.URL, proxyURL: "http://proxy.local"} - - w := httptest.NewRecorder() - h.handlePackageMetadata(w, httptest.NewRequest(http.MethodGet, "/lodash/4.17.21", nil)) - if w.Code != http.StatusForbidden { - t.Fatalf("status = %d, want 403; body: %s", w.Code, w.Body.String()) - } -} - -// `npm search` asks for /-/v1/search?text=.... The path used to fall through to -// the package dispatch, which dropped the query string, escaped the path into -// a package name and cached the answer, so every later search got the first -// one's results. -func TestNPMSearchRelaysQueryUncached(t *testing.T) { - var ( - requests int - gotPaths []string - ) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests++ - gotPaths = append(gotPaths, r.URL.EscapedPath()) - if r.URL.EscapedPath() != "/-/v1/search" { - w.WriteHeader(http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"objects":[],"total":0,"query":%q}`, r.URL.RawQuery) - })) - defer upstream.Close() - - proxy, db, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - - for _, query := range []string{"text=foo&size=20", "text=bar&size=20", "text=foo&size=20"} { - w := serveNPM(t, h, http.MethodGet, "/-/v1/search?"+query, nil, nil) - if w.Code != http.StatusOK { - t.Fatalf("search %q: status = %d; body: %s", query, w.Code, w.Body.String()) - } - if want := fmt.Sprintf(`{"objects":[],"total":0,"query":%q}`, query); w.Body.String() != want { - t.Errorf("search %q: body = %s, want %s", query, w.Body.String(), want) - } - } - if requests != 3 { - t.Errorf("upstream requests = %d (%q), want 3: searches must not be cached", requests, gotPaths) - } - for _, key := range []string{"-/v1/search", "-", "-/v1"} { - if entry, _ := db.GetMetadataCache("npm", key); entry != nil { - t.Errorf("search response cached under %q", key) - } - } -} - -// `npm ping` reads /-/ping, which used to be looked up as a package. -func TestNPMPingRelaysUpstream(t *testing.T) { - var gotPath, gotQuery string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotPath, gotQuery = r.URL.EscapedPath(), r.URL.RawQuery - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, "{}") - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - - w := serveNPM(t, h, http.MethodGet, "/-/ping?write=true", nil, nil) - if w.Code != http.StatusOK || w.Body.String() != "{}" { - t.Fatalf("ping = %d %q, want 200 {}", w.Code, w.Body.String()) - } - if gotPath != "/-/ping" || gotQuery != "write=true" { - t.Errorf("upstream request = %q?%q, want /-/ping?write=true", gotPath, gotQuery) - } -} - -// TestNPMHeadPackument checks that packuments answer HEAD, which pnpm 12 -// sends before fetching each one, while tarballs and registry endpoints -// still only answer GET. -func TestNPMHeadPackument(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/testpkg" { - w.WriteHeader(http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"name":"testpkg","versions":{"1.0.0":{"name":"testpkg","version":"1.0.0","dist":{"tarball":"https://registry.npmjs.org/testpkg/-/testpkg-1.0.0.tgz"}}}}`)) - })) - defer upstream.Close() - h := &NPMHandler{proxy: testProxy(), upstreamURL: upstream.URL, proxyURL: "http://proxy.local"} - - get := serveNPM(t, h, http.MethodGet, "/testpkg", nil, nil) - head := serveNPM(t, h, http.MethodHead, "/testpkg", nil, nil) - if head.Code != http.StatusOK || head.Header().Get("ETag") == "" || head.Header().Get("ETag") != get.Header().Get("ETag") { - t.Errorf("HEAD /testpkg = %d, ETag %q; want 200 with GET's ETag %q", head.Code, head.Header().Get("ETag"), get.Header().Get("ETag")) - } - - for _, path := range []string{"/testpkg/-/testpkg-1.0.0.tgz", "/-/ping", "/-/v1/search", "/-/package/testpkg/dist-tags"} { - if w := serveNPM(t, h, http.MethodHead, path, nil, nil); w.Code != http.StatusMethodNotAllowed { - t.Errorf("HEAD %s = %d, want %d", path, w.Code, http.StatusMethodNotAllowed) - } - } -} diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index 7a5b1b2..40b8b5f 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -4,41 +4,22 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "strings" + "time" - "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" ) const ( - nugetUpstream = "https://api.nuget.org" - nugetSearchUpstream = "https://azuresearch-usnc.nuget.org" + nugetUpstream = "https://api.nuget.org" ) -// Packages are stored under pkg:nuget/{id} with the id lowercased, since -// NuGet ids are case-insensitive. purl keeps nuget names as written, so -// package rules go through its comparison form, which lowercases them the -// same way, before the default mapping. -func init() { - retention.Default.Register(retention.Spec{Key: "nuget", CanonicalPackage: nugetRetentionKey}) -} - -// nugetRetentionKey maps a package rule to the stored package PURL, so -// pkg:nuget/Newtonsoft.Json matches the stored pkg:nuget/newtonsoft.json. -func nugetRetentionKey(p *purl.PURL) (string, bool) { - if p == nil { - return "", false - } - c := p.ComparisonForm() - return retention.DefaultCanonical("nuget")(purl.New(c.Type, c.Namespace, c.Name, "", nil)) -} - // NuGetHandler handles NuGet V3 API protocol requests. type NuGetHandler struct { proxy *Proxy upstreamURL string - searchURL string proxyURL string } @@ -47,20 +28,10 @@ func NewNuGetHandler(proxy *Proxy, proxyURL string) *NuGetHandler { return &NuGetHandler{ proxy: proxy, upstreamURL: nugetUpstream, - searchURL: nugetSearchUpstream, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } -// NewNuGetHandlerWithUpstreams creates a NuGet handler with custom API and -// search upstreams. -func NewNuGetHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, searchURL string) *NuGetHandler { - h := NewNuGetHandler(proxy, proxyURL) - h.upstreamURL = configuredUpstreamURL(upstreamURL, nugetUpstream) - h.searchURL = configuredUpstreamURL(searchURL, nugetSearchUpstream) - return h -} - // Routes returns the HTTP handler for NuGet requests. func (h *NuGetHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -70,12 +41,10 @@ func (h *NuGetHandler) Routes() http.Handler { // Package content (downloads) mux.HandleFunc("GET /v3-flatcontainer/{id}/{version}/{filename}", h.handleDownload) - mux.HandleFunc("GET /v3-flatcontainer/{id}/index.json", h.handleVersionList) + mux.HandleFunc("GET /v3-flatcontainer/{id}/index.json", h.proxyUpstream) // Registration (package metadata) - use prefix matching since {version}.json isn't allowed - for _, prefix := range nugetRegistrationPrefixes { - mux.HandleFunc("GET "+prefix, h.handleRegistration) - } + mux.HandleFunc("GET /v3/registration5-gz-semver2/", h.handleRegistration) // Search mux.HandleFunc("GET /query", h.proxyUpstream) @@ -88,7 +57,7 @@ func (h *NuGetHandler) Routes() http.Handler { // handleServiceIndex serves the NuGet V3 service index with rewritten URLs. func (h *NuGetHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request) { - h.proxy.Logger.Debug("nuget service index request") + h.proxy.Logger.Info("nuget service index request") upstreamURL := h.upstreamURL + "/v3/index.json" @@ -98,9 +67,6 @@ func (h *NuGetHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request http.Error(w, "not found", http.StatusNotFound) return } - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return @@ -108,17 +74,13 @@ func (h *NuGetHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request rewritten, err := h.rewriteServiceIndex(body) if err != nil { - if h.cooldownEnabled() { - h.nugetMetadataError(w, err) - return - } h.proxy.Logger.Warn("failed to rewrite service index, proxying original", "error", err) - w.Header().Set(headerContentType, "application/json") + w.Header().Set("Content-Type", "application/json") _, _ = w.Write(body) return } - w.Header().Set(headerContentType, "application/json") + w.Header().Set("Content-Type", "application/json") _, _ = w.Write(rewritten) } @@ -143,35 +105,189 @@ func (h *NuGetHandler) rewriteServiceIndex(body []byte) ([]byte, error) { id, _ := rmap["@id"].(string) rtype, _ := rmap["@type"].(string) - // Rewrite URLs for services we proxy. The service type determines the - // local route because an upstream index may advertise a different host. - if id != "" { - rmap["@id"] = h.rewriteNuGetURL(id, rtype) + // Rewrite URLs for services we proxy + if id != "" && h.shouldRewriteService(rtype) { + newURL := h.rewriteNuGetURL(id) + rmap["@id"] = newURL } } return json.Marshal(index) } -// rewriteNuGetURL rewrites a NuGet service URL based on its advertised type. -// Service types the proxy does not handle are returned unchanged. -func (h *NuGetHandler) rewriteNuGetURL(origURL, serviceType string) string { - switch serviceType { - case "PackageBaseAddress/3.0.0": - return h.proxyURL + "/nuget/v3-flatcontainer/" - case "RegistrationsBaseUrl", "RegistrationsBaseUrl/3.0.0-beta", "RegistrationsBaseUrl/3.0.0-rc": - return h.proxyURL + "/nuget/v3/registration5-semver1/" - case "RegistrationsBaseUrl/3.4.0": - return h.proxyURL + "/nuget/v3/registration5-gz-semver1/" - case "RegistrationsBaseUrl/3.6.0", "RegistrationsBaseUrl/Versioned": - return h.proxyURL + "/nuget/v3/registration5-gz-semver2/" - case "SearchQueryService", "SearchQueryService/3.0.0-rc", "SearchQueryService/3.5.0": - return h.proxyURL + "/nuget/query" - case "SearchAutocompleteService", "SearchAutocompleteService/3.5.0": - return h.proxyURL + "/nuget/autocomplete" - default: - return origURL +// shouldRewriteService returns true if the service type should be rewritten. +func (h *NuGetHandler) shouldRewriteService(serviceType string) bool { + // Rewrite package content and registration services + rewriteTypes := []string{ + "PackageBaseAddress/3.0.0", + "RegistrationsBaseUrl/3.6.0", + "RegistrationsBaseUrl/Versioned", + "SearchQueryService", + "SearchQueryService/3.0.0-rc", + "SearchQueryService/3.5.0", + "SearchAutocompleteService", + "SearchAutocompleteService/3.5.0", } + + for _, t := range rewriteTypes { + if serviceType == t { + return true + } + } + return false +} + +// rewriteNuGetURL rewrites a NuGet API URL to point at this proxy. +func (h *NuGetHandler) rewriteNuGetURL(origURL string) string { + // Map known NuGet API endpoints to our proxy paths + replacements := map[string]string{ + "https://api.nuget.org/v3-flatcontainer/": h.proxyURL + "/nuget/v3-flatcontainer/", + "https://api.nuget.org/v3/registration5-gz-semver2/": h.proxyURL + "/nuget/v3/registration5-gz-semver2/", + "https://azuresearch-usnc.nuget.org/query": h.proxyURL + "/nuget/query", + "https://azuresearch-usnc.nuget.org/autocomplete": h.proxyURL + "/nuget/autocomplete", + } + + for old, new := range replacements { + if strings.HasPrefix(origURL, old) { + return strings.Replace(origURL, old, new, 1) + } + } + + return origURL +} + +// handleRegistration proxies NuGet registration pages, applying cooldown filtering. +func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request) { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + h.proxyUpstream(w, r) + return + } + + upstreamURL := h.buildUpstreamURL(r) + + h.proxy.Logger.Debug("fetching registration for cooldown filtering", "url", upstreamURL) + + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) + if err != nil { + http.Error(w, "failed to create request", http.StatusInternalServerError) + return + } + req.Header.Set(headerAcceptEncoding, "gzip") + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + h.proxy.Logger.Error("upstream request failed", "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + return + } + + body, err := h.proxy.ReadMetadata(resp.Body) + if err != nil { + http.Error(w, "failed to read response", http.StatusInternalServerError) + return + } + + filtered, err := h.applyCooldownFiltering(body) + if err != nil { + h.proxy.Logger.Warn("failed to filter registration, proxying original", "error", err) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + return + } + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(filtered) +} + +// applyCooldownFiltering filters versions from NuGet registration pages +// that are too recently published. +func (h *NuGetHandler) applyCooldownFiltering(body []byte) ([]byte, error) { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return body, nil + } + + var registration map[string]any + if err := json.Unmarshal(body, ®istration); err != nil { + return nil, err + } + + pages, ok := registration["items"].([]any) + if !ok { + return body, nil + } + + for _, page := range pages { + pageMap, ok := page.(map[string]any) + if !ok { + continue + } + + items, ok := pageMap["items"].([]any) + if !ok { + continue + } + + filtered := items[:0] + for _, item := range items { + itemMap, ok := item.(map[string]any) + if !ok { + continue + } + + catalogEntry, ok := itemMap["catalogEntry"].(map[string]any) + if !ok { + filtered = append(filtered, item) + continue + } + + version, _ := catalogEntry["version"].(string) + id, _ := catalogEntry["id"].(string) + publishedStr, _ := catalogEntry["published"].(string) + + if publishedStr == "" { + filtered = append(filtered, item) + continue + } + + publishedAt, err := time.Parse(time.RFC3339, publishedStr) + if err != nil { + // NuGet uses a slightly non-standard format, try parsing with fractional seconds + publishedAt, err = time.Parse("2006-01-02T15:04:05.999-07:00", publishedStr) + if err != nil { + filtered = append(filtered, item) + continue + } + } + + packagePURL := purl.MakePURLString("nuget", strings.ToLower(id), "") + + if !h.proxy.Cooldown.IsAllowed("nuget", packagePURL, publishedAt) { + h.proxy.Logger.Info("cooldown: filtering nuget version", + "package", id, "version", version, + "published", publishedStr) + continue + } + + filtered = append(filtered, item) + } + + pageMap["items"] = filtered + pageMap["count"] = len(filtered) + } + + return json.Marshal(registration) } // handleDownload serves a package file, fetching and caching from upstream if needed. @@ -185,25 +301,13 @@ func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - if h.cooldownEnabled() { - allowed, err := h.nugetDownloadAllowed(r.Context(), id, version) - if err != nil { - h.nugetMetadataError(w, err) - return - } - if !allowed { - JSONError(w, http.StatusNotFound, "version not found") - return - } - } - // Only cache .nupkg files if !strings.HasSuffix(filename, ".nupkg") { h.proxyUpstream(w, r) return } - h.proxy.Logger.Debug("nuget download request", + h.proxy.Logger.Info("nuget download request", "id", id, "version", version, "filename", filename) // NuGet package IDs are case-insensitive, lowercase for storage @@ -212,11 +316,12 @@ func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "nuget", name, version, filename, upstreamURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch package", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) + ServeArtifact(w, result) } // proxyUpstream forwards a request to NuGet without caching. @@ -239,16 +344,20 @@ func (h *NuGetHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { resp, err := h.proxy.HTTPClient.Do(req) if err != nil { - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return } defer func() { _ = resp.Body.Close() }() - h.proxy.relayResponse(w, r, resp, nil) + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) } // buildUpstreamURL constructs the upstream URL for a request. @@ -257,7 +366,7 @@ func (h *NuGetHandler) buildUpstreamURL(r *http.Request) string { // Handle query and autocomplete which go to azuresearch if strings.HasPrefix(path, "/query") || strings.HasPrefix(path, "/autocomplete") { - return h.searchURL + path + "?" + r.URL.RawQuery + return "https://azuresearch-usnc.nuget.org" + path + "?" + r.URL.RawQuery } return h.upstreamURL + path diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go deleted file mode 100644 index b9345c5..0000000 --- a/internal/handler/nuget_cooldown.go +++ /dev/null @@ -1,377 +0,0 @@ -package handler - -import ( - "bytes" - "compress/gzip" - "context" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "slices" - "strings" - "time" -) - -var nugetRegistrationPrefixes = []string{ - "/v3/registration5-semver1/", - "/v3/registration5-gz-semver1/", - "/v3/registration5-gz-semver2/", -} - -var nugetArtifactPrefixes = append([]string{"/v3-flatcontainer/"}, nugetRegistrationPrefixes...) - -const nugetRegistrationPath = "/v3/registration5-gz-semver2/" - -func (h *NuGetHandler) cooldownEnabled() bool { - return h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() -} - -func (h *NuGetHandler) nugetCooldownApplies(id string) bool { - return h.cooldownEnabled() && h.proxy.Cooldown.For("nuget", canonicalPackagePURL("nuget", strings.ToLower(id))) > 0 -} - -// Cache upstream documents, not filtered results, so policy changes and elapsed -// time take effect even while metadata is fresh. Include the upstream in the key. -func (h *NuGetHandler) nugetMetadata(ctx context.Context, path string) (map[string]any, error) { - target := h.upstreamURL + path - key := fmt.Sprintf("_cooldown/%x", sha256.Sum256([]byte(target))) - var document map[string]any - validate := func(body []byte) error { - var err error - document, err = h.decodeNuGetMetadata(body) - return err - } - _, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "nuget", key, target, "", validate) - if err != nil { - return nil, err - } - return document, nil -} - -func (h *NuGetHandler) decodeNuGetMetadata(body []byte) (map[string]any, error) { - // Normally the HTTP transport decodes gzip. Also support compressed cached - // bytes and clients with transparent decompression disabled, with the same - // metadata limit applied to the decompressed document. - if bytes.HasPrefix(body, []byte{0x1f, 0x8b}) { - reader, err := gzip.NewReader(bytes.NewReader(body)) - if err != nil { - return nil, err - } - defer func() { _ = reader.Close() }() - body, err = h.proxy.ReadMetadata(reader) - if err != nil { - return nil, err - } - } - var document map[string]any - if err := json.Unmarshal(body, &document); err != nil { - return nil, fmt.Errorf("parsing NuGet metadata: %w", err) - } - if document == nil { - return nil, fmt.Errorf("empty NuGet metadata") - } - return document, nil -} - -// Prefer semver2, but a configured source may advertise only an older hive. -// Retry only advertised aliases on 404; transport/validation errors must not -// silently switch to a hive with less complete metadata. Keep requests on the -// configured upstream, consistent with the service-index route rewriting. -func (h *NuGetHandler) nugetRegistrationMetadata(ctx context.Context, suffix string) (map[string]any, string, error) { - path := nugetRegistrationPath + suffix - document, err := h.nugetMetadata(ctx, path) - if !errors.Is(err, ErrUpstreamNotFound) { - return document, path, err - } - index, indexErr := h.nugetMetadata(ctx, "/v3/index.json") - if indexErr != nil { - return nil, path, indexErr - } - resources, _ := index["resources"].([]any) - seen := map[string]bool{nugetRegistrationPath: true} - for _, resource := range resources { - entry, _ := resource.(map[string]any) - service, _ := entry["@type"].(string) - id, _ := entry["@id"].(string) - if id == "" || !strings.HasPrefix(service, "RegistrationsBaseUrl") { - continue - } - prefix := strings.TrimPrefix(h.rewriteNuGetURL(id, service), h.proxyURL+"/nuget") - if !slices.Contains(nugetRegistrationPrefixes, prefix) || seen[prefix] { - continue - } - seen[prefix] = true - path = prefix + suffix - document, err = h.nugetMetadata(ctx, path) - if !errors.Is(err, ErrUpstreamNotFound) { - return document, path, err - } - } - return nil, path, err -} - -func (h *NuGetHandler) nugetMetadataError(w http.ResponseWriter, err error) { - if errors.Is(err, ErrUpstreamNotFound) { - JSONError(w, http.StatusNotFound, "package metadata not found") - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Warn("failed to process NuGet metadata", "error", err) - JSONError(w, http.StatusBadGateway, "failed to process package metadata") -} - -func (h *NuGetHandler) handleVersionList(w http.ResponseWriter, r *http.Request) { - if !h.cooldownEnabled() { - h.proxyUpstream(w, r) - return - } - id := strings.ToLower(r.PathValue("id")) - document, err := h.nugetMetadata(r.Context(), "/v3-flatcontainer/"+url.PathEscape(id)+"/index.json") - if err != nil { - h.nugetMetadataError(w, err) - return - } - blocked := make(map[string]bool) - // A globally enabled policy may still exempt this package or ecosystem. - // Keep metadata caching, but do not require publication data in that case. - if h.nugetCooldownApplies(id) { - registration, registrationPath, err := h.nugetRegistrationMetadata(r.Context(), url.PathEscape(id)+"/index.json") - if err == nil { - err = h.expandNuGetPages(r.Context(), registration, registrationPath) - } - if err != nil { - h.nugetMetadataError(w, err) - return - } - h.collectNuGetBlockedVersions(registration, id, blocked) - } - versions, ok := document["versions"].([]any) - if !ok { - h.nugetMetadataError(w, fmt.Errorf("missing NuGet versions")) - return - } - filtered := make([]any, 0, len(versions)) - for _, value := range versions { - version, ok := value.(string) - if !ok { - h.nugetMetadataError(w, fmt.Errorf("invalid NuGet version")) - return - } - if !blocked[nugetVersionKey(version)] { - filtered = append(filtered, value) - } - } - document["versions"] = filtered - w.Header().Set(headerContentType, contentTypeJSON) - _ = json.NewEncoder(w).Encode(document) -} - -func nugetVersionKey(version string) string { - version, _, _ = strings.Cut(version, "+") - return strings.ToLower(version) -} - -func (h *NuGetHandler) nugetDownloadAllowed(ctx context.Context, id, version string) (bool, error) { - if !h.nugetCooldownApplies(id) { - return true, nil - } - suffix := url.PathEscape(strings.ToLower(id)) + "/" + url.PathEscape(nugetVersionKey(version)) + ".json" - leaf, _, err := h.nugetRegistrationMetadata(ctx, suffix) - if err != nil { - return false, err - } - return h.nugetLeafAllowed(leaf, id), nil -} - -// A standalone leaf has published at its root; leaves embedded in pages carry -// it in catalogEntry. Missing/invalid timestamps retain the existing permissive -// behavior, but fetch and JSON errors must not bypass the policy. -func (h *NuGetHandler) nugetLeafAllowed(leaf map[string]any, id string) bool { - if !h.cooldownEnabled() { - return true - } - entry := nugetCatalogEntry(leaf) - if id == "" { - id, _ = entry["id"].(string) - } - published, _ := entry["published"].(string) - when, err := time.Parse(time.RFC3339, published) - if err != nil { - return true - } - return h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) -} - -func nugetCatalogEntry(leaf map[string]any) map[string]any { - if entry, ok := leaf["catalogEntry"].(map[string]any); ok { - return entry - } - return leaf -} - -func (h *NuGetHandler) collectNuGetBlockedVersions(document map[string]any, id string, blocked map[string]bool) { - entry := nugetCatalogEntry(document) - if version, ok := entry["version"].(string); ok && !h.nugetLeafAllowed(document, id) { - blocked[nugetVersionKey(version)] = true - } - items, _ := document["items"].([]any) - for _, item := range items { - if child, ok := item.(map[string]any); ok { - h.collectNuGetBlockedVersions(child, id, blocked) - } - } -} - -func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request) { - if !h.cooldownEnabled() { - h.proxyUpstream(w, r) - return - } - id := nugetRegistrationID(r.URL.Path) - applyCooldown := h.nugetCooldownApplies(id) - document, err := h.nugetMetadata(r.Context(), r.URL.Path) - if err == nil && applyCooldown { - err = h.expandNuGetPages(r.Context(), document, r.URL.Path) - } - if err != nil { - h.nugetMetadataError(w, err) - return - } - _, hasItems := document["items"] - if applyCooldown && !h.filterNuGetRegistration(document, id) && !hasItems { - JSONError(w, http.StatusNotFound, "version not found") - return - } - h.rewriteNuGetRegistrationLinks(document) - w.Header().Set(headerContentType, contentTypeJSON) - _ = json.NewEncoder(w).Encode(document) -} - -func nugetRegistrationID(path string) string { - for _, prefix := range nugetRegistrationPrefixes { - if rest, ok := strings.CutPrefix(path, prefix); ok { - id, _, _ := strings.Cut(rest, "/") - return id - } - } - return "" -} - -// Only expand index pages, never recursively follow arbitrary upstream links. -// Pin requests to this configured upstream and the current package's page path. -func (h *NuGetHandler) expandNuGetPages(ctx context.Context, document map[string]any, path string) error { - if !strings.HasSuffix(path, "/index.json") { - return nil - } - items, ok := document["items"].([]any) - if !ok { - return fmt.Errorf("missing registration pages") - } - base, err := url.Parse(h.upstreamURL + path) - if err != nil { - return err - } - pagePrefix := strings.TrimSuffix(base.Path, "index.json") + "page/" - for _, item := range items { - page, ok := item.(map[string]any) - if !ok { - return fmt.Errorf("invalid registration page") - } - if _, ok := page["items"].([]any); ok { - continue - } - link, _ := page["@id"].(string) - target, err := base.Parse(link) - if err != nil || target.Scheme != base.Scheme || target.Host != base.Host || - !strings.HasPrefix(target.Path, pagePrefix) || containsPathTraversal(target.Path) || target.RawQuery != "" || target.Fragment != "" { - return fmt.Errorf("invalid registration page URL: %q", link) - } - upstream, _ := url.Parse(h.upstreamURL) - pageDocument, err := h.nugetMetadata(ctx, strings.TrimPrefix(target.Path, upstream.Path)) - if err != nil { - return err - } - leaves, ok := pageDocument["items"].([]any) - if !ok { - return fmt.Errorf("missing registration leaves") - } - page["items"] = leaves - } - return nil -} - -func (h *NuGetHandler) filterNuGetRegistration(document map[string]any, id string) bool { - items, ok := document["items"].([]any) - if !ok { - return h.nugetLeafAllowed(document, id) - } - filtered := make([]any, 0, len(items)) - for _, item := range items { - child, ok := item.(map[string]any) - if ok && h.filterNuGetRegistration(child, id) { - filtered = append(filtered, child) - } - } - document["items"] = filtered - document["count"] = len(filtered) - // Page bounds describe the retained leaves, not versions hidden by cooldown. - if _, isPage := document["lower"]; isPage && len(filtered) > 0 { - first, _ := filtered[0].(map[string]any) - last, _ := filtered[len(filtered)-1].(map[string]any) - document["lower"] = nugetCatalogEntry(first)["version"] - document["upper"] = nugetCatalogEntry(last)["version"] - } - return len(filtered) > 0 -} - -func (h *NuGetHandler) rewriteNuGetRegistrationLinks(value any) { - switch node := value.(type) { - case map[string]any: - for key, child := range node { - if link, ok := child.(string); ok { - switch key { - case "@id", "parent", "registration", "packageContent": - node[key] = h.nugetProxyLink(link) - } - } else { - h.rewriteNuGetRegistrationLinks(child) - } - } - case []any: - for _, child := range node { - h.rewriteNuGetRegistrationLinks(child) - } - } -} - -func (h *NuGetHandler) nugetProxyLink(link string) string { - u, err := url.Parse(link) - if err != nil { - return link - } - upstream, err := url.Parse(h.upstreamURL) - if err != nil { - return link - } - path := u.Path - if u.Host == upstream.Host { - path = strings.TrimPrefix(path, upstream.Path) - } - for _, prefix := range nugetArtifactPrefixes { - if strings.HasPrefix(path, prefix) { - proxy, err := url.Parse(h.proxyURL + "/nuget" + path) - if err != nil { - return link - } - proxy.RawQuery = u.RawQuery - proxy.Fragment = u.Fragment - return proxy.String() - } - } - return link -} diff --git a/internal/handler/nuget_cooldown_test.go b/internal/handler/nuget_cooldown_test.go deleted file mode 100644 index 3160bd5..0000000 --- a/internal/handler/nuget_cooldown_test.go +++ /dev/null @@ -1,555 +0,0 @@ -package handler - -import ( - "bytes" - "compress/gzip" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/registries/fetch" -) - -func TestNuGetCooldownRoutes(t *testing.T) { - for _, disableCompression := range []bool{false, true} { - t.Run(map[bool]string{false: "transport gzip", true: "explicit gzip"}[disableCompression], func(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - proxy.Cooldown = &cooldown.Config{Default: "14d"} - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - seedPackage(t, db, store, "nuget", "testpkg", "2.0.0", "testpkg.2.0.0.nupkg", "cached package") - seedPackage(t, db, store, "nuget", "testpkg", "1.0.0", "testpkg.1.0.0.nupkg", "old package") - metadataRequests := 0 - upstream := newNuGetCooldownUpstream(t, &metadataRequests) - defer upstream.Close() - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.DisableCompression = disableCompression - defer transport.CloseIdleConnections() - proxy.HTTPClient = &http.Client{Transport: transport} - h := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) - routes := http.StripPrefix("/nuget", h.Routes()) - get := func(path string, status int) *httptest.ResponseRecorder { - t.Helper() - return nugetGet(t, routes, path, status) - } - list := get("/nuget/v3-flatcontainer/testpkg/index.json", http.StatusOK) - if got := strings.TrimSpace(list.Body.String()); got != `{"versions":["1.0.0"]}` { - t.Fatalf("filtered list = %s", got) - } - index := get("/nuget"+nugetRegistrationPath+"testpkg/index.json", http.StatusOK) - if strings.Contains(index.Body.String(), `"version":"2.0.0"`) || strings.Contains(index.Body.String(), upstream.URL) { - t.Fatalf("registration leaks blocked leaf or upstream link: %s", index.Body.String()) - } - if index.Header().Get("Content-Encoding") != "" || !json.Valid(index.Body.Bytes()) { - t.Fatal("registration must be decoded JSON") - } - var doc struct { - Items []struct { - ID string `json:"@id"` - Count int - Lower, Upper string - Items []struct { - ID string `json:"@id"` - PackageContent string - } - } - } - if err := json.Unmarshal(index.Body.Bytes(), &doc); err != nil { - t.Fatal(err) - } - if len(doc.Items) != 1 || doc.Items[0].Count != 1 || doc.Items[0].Upper != "1.0.0" { - t.Fatalf("incorrect page: %+v", doc) - } - get(doc.Items[0].ID, http.StatusOK) - get(doc.Items[0].Items[0].ID, http.StatusOK) - get(doc.Items[0].Items[0].PackageContent, http.StatusOK) - get("/nuget"+nugetRegistrationPath+"testpkg/2.0.0.json", http.StatusNotFound) - get("/nuget/v3-flatcontainer/TestPkg/2.0.0/testpkg.2.0.0.nupkg", http.StatusNotFound) - get("/nuget/v3-flatcontainer/testpkg/2.0.0/testpkg.nuspec", http.StatusNotFound) - if fetcher.fetchCalled { - t.Fatal("blocked or cached downloads must not fetch artifacts") - } - - // Reevaluate fresh, unfiltered metadata under a changed package policy. - requestsBefore := metadataRequests - proxy.Cooldown = &cooldown.Config{Default: "14d", Packages: map[string]string{"pkg:nuget/testpkg": "1d"}} - list = get("/nuget/v3-flatcontainer/testpkg/index.json", http.StatusOK) - if !strings.Contains(list.Body.String(), "2.0.0") { - t.Fatal("fresh metadata retained the previous policy") - } - get("/nuget/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", http.StatusOK) - if metadataRequests != requestsBefore { - t.Fatal("fresh metadata should be reused") - } - }) - } -} - -func nugetGet(t *testing.T, routes http.Handler, path string, status int) *httptest.ResponseRecorder { - t.Helper() - w := httptest.NewRecorder() - routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - if w.Code != status { - t.Fatalf("GET %s: status %d, want %d: %s", path, w.Code, status, w.Body.String()) - } - return w -} - -func TestNuGetMetadataWithoutEffectiveCooldown(t *testing.T) { - for _, tt := range []struct { - name string - policy *cooldown.Config - }{ - {"package exemption", &cooldown.Config{Default: "14d", Packages: map[string]string{"pkg:nuget/testpkg": "0"}}}, - {"ecosystem exemption", &cooldown.Config{Default: "14d", Ecosystems: map[string]string{"nuget": "0"}}}, - {"other ecosystem only", &cooldown.Config{Ecosystems: map[string]string{"npm": "14d"}}}, - {"other package only", &cooldown.Config{Packages: map[string]string{"pkg:nuget/other": "14d"}}}, - } { - t.Run(tt.name, func(t *testing.T) { - const body = `{"versions":["1.0.0","2.0.0"]}` - const pagePath = nugetRegistrationPath + "testpkg/page/1.0.0/2.0.0.json" - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/v3-flatcontainer/testpkg/index.json": - _, _ = io.WriteString(w, body) - case nugetRegistrationPath + "testpkg/index.json": - _, _ = io.WriteString(w, `{"count":1,"items":[{"@id":"`+pagePath+`","count":2,"lower":"1.0.0","upper":"2.0.0"}]}`) - default: - t.Errorf("unnecessary registration request: %s", r.URL.Path) - http.Error(w, "registration unavailable", http.StatusServiceUnavailable) - } - })) - defer upstream.Close() - p := nugetTestProxy() - p.Cooldown = tt.policy - h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) - w := nugetGet(t, h.Routes(), "/v3-flatcontainer/TestPkg/index.json", http.StatusOK) - if got := strings.TrimSpace(w.Body.String()); got != body { - t.Fatalf("version list = %s, want %s", got, body) - } - w = nugetGet(t, h.Routes(), nugetRegistrationPath+"testpkg/index.json", http.StatusOK) - if !strings.Contains(w.Body.String(), `"@id":"http://proxy.test/nuget`+pagePath+`"`) { - t.Fatalf("registration page link was not rewritten: %s", w.Body.String()) - } - }) - } -} - -func TestNuGetCooldownColdDownload(t *testing.T) { - for _, tt := range []struct { - name, published string - policy *cooldown.Config - want int - }{ - {"recent", time.Now().Add(-time.Hour).Format(time.RFC3339), &cooldown.Config{Default: "14d"}, http.StatusNotFound}, - {"missing timestamp", "", &cooldown.Config{Default: "14d"}, http.StatusOK}, - {"package exemption", time.Now().Add(-time.Hour).Format(time.RFC3339), &cooldown.Config{Default: "14d", Packages: map[string]string{"pkg:nuget/testpkg": "0"}}, http.StatusOK}, - {"ecosystem override", time.Now().Add(-time.Hour).Format(time.RFC3339), &cooldown.Config{Ecosystems: map[string]string{"nuget": "14d"}}, http.StatusNotFound}, - } { - t.Run(tt.name, func(t *testing.T) { - p, _, _, fetcher := setupTestProxy(t) - p.Cooldown = tt.policy - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package")), ContentType: "application/octet-stream"} - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _ = json.NewEncoder(w).Encode(map[string]string{"published": tt.published}) - })) - defer upstream.Close() - h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) - nugetGet(t, h.Routes(), "/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", tt.want) - if fetcher.fetchCalled != (tt.want == http.StatusOK) { - t.Errorf("artifact fetch called = %v", fetcher.fetchCalled) - } - }) - } -} - -// TestNuGetCooldownConcurrentDownloads asserts that a download which joins -// another request's metadata fetch still applies the cooldown. Each caller -// decodes the leaf in its own validate callback, so a waiter on the shared -// fetch must run its own rather than finding no document and allowing it. -func TestNuGetCooldownConcurrentDownloads(t *testing.T) { - p, db, store, fetcher := setupTestProxy(t) - p.Cooldown = &cooldown.Config{Default: "14d"} - seedPackage(t, db, store, "nuget", "testpkg", "2.0.0", "testpkg.2.0.0.nupkg", "cached package") - entered, release := make(chan struct{}), make(chan struct{}) - var enteredOnce, releaseOnce sync.Once - releaseUpstream := func() { releaseOnce.Do(func() { close(release) }) } - var calls atomic.Int64 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) - enteredOnce.Do(func() { close(entered) }) - <-release - _ = json.NewEncoder(w).Encode(map[string]string{"published": time.Now().Add(-time.Hour).Format(time.RFC3339)}) - })) - // Cleanups run last first: release the held request before closing the - // server, so a failed wait can't hang the test. - t.Cleanup(upstream.Close) - t.Cleanup(releaseUpstream) - routes := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL).Routes() - - statuses := make(chan int, 2) - download := func() { - w := httptest.NewRecorder() - routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", nil)) - statuses <- w.Code - } - go download() - <-entered - go download() - target := upstream.URL + nugetRegistrationPath + "testpkg/2.0.0.json" - key := metadataCoalesceKey("nuget", fmt.Sprintf("_cooldown/%x", sha256.Sum256([]byte(target))), target, contentTypeJSON, "", true) - waitForMetadataWaiters(t, p, key, 1) - releaseUpstream() - - for range 2 { - if status := <-statuses; status != http.StatusNotFound { - t.Errorf("download status = %d, want %d", status, http.StatusNotFound) - } - } - if got := calls.Load(); got != 1 { - t.Errorf("upstream metadata requests = %d, want 1", got) - } - if fetcher.fetchCalled { - t.Error("blocked downloads must not fetch artifacts") - } -} - -func TestNuGetRegistrationServiceAliases(t *testing.T) { - h := NewNuGetHandler(nugetTestProxy(), "http://proxy.test") - for _, tt := range []struct{ service, path string }{ - {"RegistrationsBaseUrl", "/v3/registration5-semver1/"}, - {"RegistrationsBaseUrl/3.0.0-beta", "/v3/registration5-semver1/"}, - {"RegistrationsBaseUrl/3.0.0-rc", "/v3/registration5-semver1/"}, - {"RegistrationsBaseUrl/3.4.0", "/v3/registration5-gz-semver1/"}, - {"RegistrationsBaseUrl/3.6.0", nugetRegistrationPath}, - {"RegistrationsBaseUrl/Versioned", nugetRegistrationPath}, - } { - t.Run(tt.service, func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != tt.path+"testpkg/index.json" { - t.Errorf("wrong hive: %s", r.URL.Path) - } - _, _ = io.WriteString(w, `{"count":0,"items":[]}`) - })) - defer upstream.Close() - h.upstreamURL = upstream.URL - h.proxy.Cooldown = &cooldown.Config{Default: "14d"} - body := []byte(`{"resources":[{"@id":"` + upstream.URL + tt.path + `","@type":"` + tt.service + `"}]}`) - out, err := h.rewriteServiceIndex(body) - if err != nil { - t.Fatal(err) - } - var doc struct { - Resources []struct { - ID string `json:"@id"` - } - } - if err := json.Unmarshal(out, &doc); err != nil { - t.Fatal(err) - } - w := httptest.NewRecorder() - http.StripPrefix("/nuget", h.Routes()).ServeHTTP(w, httptest.NewRequest(http.MethodGet, doc.Resources[0].ID+"testpkg/index.json", nil)) - if w.Code != http.StatusOK { - t.Fatalf("alias route status = %d: %s", w.Code, w.Body.String()) - } - }) - } -} - -func TestNuGetCooldownMetadataErrors(t *testing.T) { - for _, tt := range []struct { - name, body string - status int - }{ - {"upstream failure", "unavailable", http.StatusServiceUnavailable}, - {"invalid JSON", "broken JSON", http.StatusOK}, - {"null", "null", http.StatusOK}, - } { - t.Run(tt.name, func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(tt.status) - _, _ = io.WriteString(w, tt.body) - })) - defer upstream.Close() - p := nugetTestProxy() - p.Cooldown = &cooldown.Config{Default: "14d"} - h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) - for _, path := range []string{"/v3-flatcontainer/testpkg/index.json", "/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", nugetRegistrationPath + "testpkg/index.json"} { - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - if w.Code != http.StatusBadGateway { - t.Errorf("GET %s: %d, want 502", path, w.Code) - } - } - }) - } -} - -func TestNuGetCooldownRejectsUnsafePageLinks(t *testing.T) { - for _, link := range []string{"https://other.example/page.json", "/v3/registration5-gz-semver2/other/page/1/2.json", "page/../index.json", "index.json"} { - t.Run(link, func(t *testing.T) { - requests := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests++ - _ = json.NewEncoder(w).Encode(map[string]any{"items": []any{map[string]any{"@id": link}}}) - })) - defer upstream.Close() - p := nugetTestProxy() - p.Cooldown = &cooldown.Config{Default: "14d"} - h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) - nugetGet(t, h.Routes(), nugetRegistrationPath+"testpkg/index.json", http.StatusBadGateway) - if requests != 1 { - t.Fatalf("unsafe page link was followed (%d requests)", requests) - } - }) - } -} - -func TestNuGetCooldownDecompressedMetadataLimit(t *testing.T) { - var compressed bytes.Buffer - gz := gzip.NewWriter(&compressed) - _, _ = io.WriteString(gz, `{"padding":"`+strings.Repeat("x", 2048)+`"}`) - _ = gz.Close() - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Encoding", "gzip") - _, _ = w.Write(compressed.Bytes()) - })) - defer upstream.Close() - p := nugetTestProxy() - p.Cooldown = &cooldown.Config{Default: "14d"} - p.MetadataMaxSize = 1024 - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.DisableCompression = true - defer transport.CloseIdleConnections() - p.HTTPClient = &http.Client{Transport: transport} - h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) - nugetGet(t, h.Routes(), nugetRegistrationPath+"testpkg/index.json", http.StatusBadGateway) -} - -func newNuGetCooldownUpstream(t *testing.T, metadataRequests *int) *httptest.Server { - t.Helper() - var upstream *httptest.Server - upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - (*metadataRequests)++ - base := upstream.URL + nugetRegistrationPath + "testpkg/" - leaf := func(version string, age time.Duration) map[string]any { - return map[string]any{ - "@id": base + version + ".json", - "packageContent": upstream.URL + "/v3-flatcontainer/testpkg/" + version + "/testpkg." + version + ".nupkg", - "catalogEntry": map[string]any{"id": "TestPkg", "version": version, "published": time.Now().Add(-age).Format(time.RFC3339)}, - } - } - page := map[string]any{"@id": base + "page/1.0.0/2.0.0.json", "lower": "1.0.0", "upper": "2.0.0", "count": 2, - "parent": base + "index.json", "items": []any{leaf("1.0.0", 30*24*time.Hour), leaf("2.0.0", 2*24*time.Hour)}} - var body any - switch r.URL.Path { - case "/v3-flatcontainer/testpkg/index.json": - body = map[string]any{"versions": []string{"1.0.0", "2.0.0"}} - case nugetRegistrationPath + "testpkg/index.json": - // This index deliberately does not inline its leaves. - body = map[string]any{"count": 1, "items": []any{map[string]any{ - "@id": page["@id"], "count": 2, "lower": "1.0.0", "upper": "2.0.0", - }}} - case nugetRegistrationPath + "testpkg/page/1.0.0/2.0.0.json": - body = page - case nugetRegistrationPath + "testpkg/1.0.0.json": - body = map[string]any{"published": time.Now().Add(-30 * 24 * time.Hour).Format(time.RFC3339)} - case nugetRegistrationPath + "testpkg/2.0.0.json": - body = map[string]any{"published": time.Now().Add(-2 * 24 * time.Hour).Format(time.RFC3339)} - default: - t.Errorf("unexpected metadata request: %s", r.URL.Path) - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Content-Encoding", "gzip") - gz := gzip.NewWriter(w) - _ = json.NewEncoder(gz).Encode(body) - _ = gz.Close() - })) - - return upstream -} - -func TestNuGetCooldownLegacyRegistration(t *testing.T) { - for _, service := range []string{"RegistrationsBaseUrl", "RegistrationsBaseUrl/3.0.0-beta", "RegistrationsBaseUrl/3.0.0-rc", "RegistrationsBaseUrl/3.4.0"} { - t.Run(service, func(t *testing.T) { - prefix := "/v3/registration5-semver1/" - if service == "RegistrationsBaseUrl/3.4.0" { - prefix = "/v3/registration5-gz-semver1/" - } - upstream := newNuGetLegacyUpstream(t, service, prefix) - defer upstream.Close() - p, db, store, fetcher := setupTestProxy(t) - p.Cooldown = &cooldown.Config{Default: "14d"} - seedPackage(t, db, store, "nuget", "testpkg", "1.0.0", "testpkg.1.0.0.nupkg", "cached old package") - seedPackage(t, db, store, "nuget", "testpkg", "2.0.0", "testpkg.2.0.0.nupkg", "cached recent package") - h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL+"/feed", upstream.URL) - list := nugetGet(t, h.Routes(), "/v3-flatcontainer/testpkg/index.json", http.StatusOK) - if strings.TrimSpace(list.Body.String()) != `{"versions":["1.0.0"]}` { - t.Fatalf("incorrect version list: %s", list.Body.String()) - } - nugetGet(t, h.Routes(), "/v3-flatcontainer/testpkg/1.0.0/testpkg.1.0.0.nupkg", http.StatusOK) - nugetGet(t, h.Routes(), "/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", http.StatusNotFound) - if fetcher.fetchCalled { - t.Fatal("cached or blocked package must not be fetched") - } - }) - } -} - -func newNuGetLegacyUpstream(t *testing.T, service, prefix string) *httptest.Server { - t.Helper() - var upstream *httptest.Server - upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - path := strings.TrimPrefix(r.URL.Path, "/feed") - base := upstream.URL + "/feed" + prefix + "testpkg/" - published := func(age time.Duration) string { return time.Now().Add(-age).Format(time.RFC3339) } - var body any - switch path { - case "/v3/index.json": - body = map[string]any{"resources": []any{map[string]string{"@id": upstream.URL + "/feed" + prefix, "@type": service}}} - case "/v3-flatcontainer/testpkg/index.json": - body = map[string]any{"versions": []string{"1.0.0", "2.0.0"}} - case prefix + "testpkg/index.json": - body = map[string]any{"items": []any{map[string]any{"@id": base + "page/1.0.0/2.0.0.json"}}} - case prefix + "testpkg/page/1.0.0/2.0.0.json": - body = map[string]any{"items": []any{ - map[string]any{"catalogEntry": map[string]string{"id": "testpkg", "version": "1.0.0", "published": published(30 * 24 * time.Hour)}}, - map[string]any{"catalogEntry": map[string]string{"id": "testpkg", "version": "2.0.0", "published": published(time.Hour)}}, - }} - case prefix + "testpkg/1.0.0.json": - body = map[string]string{"published": published(30 * 24 * time.Hour)} - case prefix + "testpkg/2.0.0.json": - body = map[string]string{"published": published(time.Hour)} - default: - if !strings.HasPrefix(path, nugetRegistrationPath) { - t.Errorf("unexpected request: %s", r.URL.Path) - } - http.NotFound(w, r) - return - } - _ = json.NewEncoder(w).Encode(body) - })) - return upstream -} - -func TestNuGetRegistrationDoesNotFallbackOnFailure(t *testing.T) { - for _, status := range []int{http.StatusServiceUnavailable, http.StatusUnauthorized, http.StatusOK} { - t.Run(http.StatusText(status), func(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != nugetRegistrationPath+"testpkg/index.json" { - t.Errorf("must not switch registration hive on failure: %s", r.URL.Path) - } - w.WriteHeader(status) - _, _ = io.WriteString(w, "invalid metadata") - })) - defer upstream.Close() - h := NewNuGetHandlerWithUpstreams(nugetTestProxy(), "http://proxy.test", upstream.URL, upstream.URL) - if _, _, err := h.nugetRegistrationMetadata(t.Context(), "testpkg/index.json"); err == nil { - t.Fatal("expected metadata error") - } - }) - } -} - -func TestNuGetMetadataPreservesValidCache(t *testing.T) { - for _, invalid := range []string{"broken JSON", "null", "[]", string([]byte{0x1f, 0x8b, 0x00})} { - t.Run(invalid, func(t *testing.T) { - const good = `{"published":"2020-01-01T00:00:00Z"}` - var response atomic.Value - response.Store(good) - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if requests.Add(1) > 1 && r.Header.Get("If-None-Match") != `"good"` { - t.Errorf("cached ETag was replaced: %s", r.Header.Get("If-None-Match")) - } - body := response.Load().(string) - etag := `"good"` - if body == invalid { - etag = `"bad"` - } - w.Header().Set("ETag", etag) - _, _ = io.WriteString(w, body) - })) - defer upstream.Close() - p, _, _, _ := setupTestProxy(t) - p.CacheMetadata = true - h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) - check := func(want string) { - t.Helper() - doc, err := h.nugetMetadata(t.Context(), nugetRegistrationPath+"testpkg/1.0.0.json") - if err != nil || doc["published"] != want { - t.Fatalf("metadata = %v, err = %v, want publication %s", doc, err, want) - } - } - check("2020-01-01T00:00:00Z") - response.Store(invalid) - check("2020-01-01T00:00:00Z") // Bad 200 must fall back without overwriting. - p.MetadataTTL = time.Hour - check("2020-01-01T00:00:00Z") // The on-disk cache must still be usable. - if requests.Load() != 2 { - t.Fatalf("requests = %d, want 2", requests.Load()) - } - p.MetadataTTL = 0 - response.Store(`{"published":"2021-01-01T00:00:00Z"}`) - check("2021-01-01T00:00:00Z") // A later valid response replaces the cache. - }) - } -} - -func TestNuGetMetadataInvalidResponseNotCached(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if requests.Add(1) == 1 { - _, _ = io.WriteString(w, "invalid JSON") - return - } - _, _ = io.WriteString(w, `{"published":"2020-01-01T00:00:00Z"}`) - })) - defer upstream.Close() - p, _, _, _ := setupTestProxy(t) - p.CacheMetadata = true - p.MetadataTTL = time.Hour - h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) - path := nugetRegistrationPath + "testpkg/1.0.0.json" - if _, err := h.nugetMetadata(t.Context(), path); err == nil { - t.Fatal("invalid response without a usable cache must fail") - } - if _, err := h.nugetMetadata(t.Context(), path); err != nil { - t.Fatalf("invalid response was cached: %v", err) - } - if requests.Load() != 2 { - t.Fatalf("requests = %d, want 2", requests.Load()) - } -} - -func TestNuGetRegistrationDoesNotGuessUnadvertisedAliases(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case nugetRegistrationPath + "testpkg/index.json": - http.NotFound(w, r) - case "/v3/index.json": - _, _ = io.WriteString(w, `{"resources":[{"@type":"UnrelatedService","@id":"https://other.example/"}]}`) - default: - t.Errorf("unadvertised endpoint requested: %s", r.URL.Path) - http.NotFound(w, r) - } - })) - defer upstream.Close() - h := NewNuGetHandlerWithUpstreams(nugetTestProxy(), "http://proxy.test", upstream.URL, upstream.URL) - _, _, err := h.nugetRegistrationMetadata(t.Context(), "testpkg/index.json") - if !errors.Is(err, ErrUpstreamNotFound) { - t.Fatalf("error = %v, want metadata not found", err) - } -} diff --git a/internal/handler/nuget_test.go b/internal/handler/nuget_test.go index 5ae77dc..b2164e5 100644 --- a/internal/handler/nuget_test.go +++ b/internal/handler/nuget_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/purl" ) func nugetTestProxy() *Proxy { @@ -92,82 +91,75 @@ func TestNuGetRewriteServiceIndex(t *testing.T) { } } +func TestNuGetShouldRewriteService(t *testing.T) { + h := &NuGetHandler{} + + rewriteTypes := []string{ + "PackageBaseAddress/3.0.0", + "RegistrationsBaseUrl/3.6.0", + "RegistrationsBaseUrl/Versioned", + "SearchQueryService", + "SearchQueryService/3.0.0-rc", + "SearchQueryService/3.5.0", + "SearchAutocompleteService", + "SearchAutocompleteService/3.5.0", + } + + for _, stype := range rewriteTypes { + if !h.shouldRewriteService(stype) { + t.Errorf("shouldRewriteService(%q) = false, want true", stype) + } + } + + noRewriteTypes := []string{ + "SomeOtherService/1.0.0", + "PackagePublish/2.0.0", + "", + "SearchQueryService/99.0.0", + } + + for _, stype := range noRewriteTypes { + if h.shouldRewriteService(stype) { + t.Errorf("shouldRewriteService(%q) = true, want false", stype) + } + } +} + func TestNuGetRewriteURL(t *testing.T) { h := &NuGetHandler{ proxyURL: "http://localhost:8080", } tests := []struct { - input string - serviceType string - want string + input string + want string }{ { "https://api.nuget.org/v3-flatcontainer/", - "PackageBaseAddress/3.0.0", "http://localhost:8080/nuget/v3-flatcontainer/", }, { "https://api.nuget.org/v3/registration5-gz-semver2/", - "RegistrationsBaseUrl/3.6.0", - "http://localhost:8080/nuget/v3/registration5-gz-semver2/", - }, - { - "https://api.nuget.org/v3/registration5-gz-semver2/", - "RegistrationsBaseUrl/Versioned", "http://localhost:8080/nuget/v3/registration5-gz-semver2/", }, { "https://azuresearch-usnc.nuget.org/query", - "SearchQueryService", - "http://localhost:8080/nuget/query", - }, - { - "https://azuresearch-usnc.nuget.org/query", - "SearchQueryService/3.0.0-rc", - "http://localhost:8080/nuget/query", - }, - { - "https://azuresearch-usnc.nuget.org/query", - "SearchQueryService/3.5.0", "http://localhost:8080/nuget/query", }, { "https://azuresearch-usnc.nuget.org/autocomplete", - "SearchAutocompleteService", - "http://localhost:8080/nuget/autocomplete", - }, - { - "https://azuresearch-usnc.nuget.org/autocomplete", - "SearchAutocompleteService/3.5.0", "http://localhost:8080/nuget/autocomplete", }, { "https://example.com/unknown", - "SomeOtherService/1.0.0", "https://example.com/unknown", }, - { - "https://api.nuget.org/v2/package", - "PackagePublish/2.0.0", - "https://api.nuget.org/v2/package", - }, - { - "https://azuresearch-usnc.nuget.org/query", - "SearchQueryService/99.0.0", - "https://azuresearch-usnc.nuget.org/query", - }, - { - "https://example.com/resource", - "", - "https://example.com/resource", - }, } for _, tt := range tests { - got := h.rewriteNuGetURL(tt.input, tt.serviceType) + got := h.rewriteNuGetURL(tt.input) if got != tt.want { - t.Errorf("rewriteNuGetURL(%q, %q) = %q, want %q", tt.input, tt.serviceType, got, tt.want) + t.Errorf("rewriteNuGetURL(%q) = %q, want %q", tt.input, got, tt.want) } } } @@ -466,7 +458,6 @@ func TestNuGetProxyUpstreamForwardsAcceptEncoding(t *testing.T) { func TestNuGetBuildUpstreamURL(t *testing.T) { h := &NuGetHandler{ upstreamURL: "https://api.nuget.org", - searchURL: "https://azuresearch-usnc.nuget.org", } tests := []struct { @@ -745,7 +736,6 @@ func TestNuGetHandleDownloadMissingFilename(t *testing.T) { func TestNuGetBuildUpstreamURLQueryPath(t *testing.T) { h := &NuGetHandler{ upstreamURL: "https://api.nuget.org", - searchURL: "https://azuresearch-usnc.nuget.org", } // Query endpoint should go to azuresearch @@ -760,7 +750,6 @@ func TestNuGetBuildUpstreamURLQueryPath(t *testing.T) { func TestNuGetBuildUpstreamURLAutocompletePath(t *testing.T) { h := &NuGetHandler{ upstreamURL: "https://api.nuget.org", - searchURL: "https://azuresearch-usnc.nuget.org", } req := httptest.NewRequest(http.MethodGet, "/autocomplete?q=new&take=10", nil) @@ -813,6 +802,11 @@ func TestNuGetCooldownFiltering(t *testing.T) { }, } + body, err := json.Marshal(registration) + if err != nil { + t.Fatal(err) + } + proxy := testProxy() proxy.Cooldown = &cooldown.Config{ Default: "3d", @@ -823,11 +817,17 @@ func TestNuGetCooldownFiltering(t *testing.T) { proxyURL: "http://localhost:8080", } - if !h.filterNuGetRegistration(registration, "") { - t.Fatal("expected registration items to be retained") + filtered, err := h.applyCooldownFiltering(body) + if err != nil { + t.Fatal(err) } - pages := registration["items"].([]any) + var result map[string]any + if err := json.Unmarshal(filtered, &result); err != nil { + t.Fatal(err) + } + + pages := result["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) @@ -841,7 +841,7 @@ func TestNuGetCooldownFiltering(t *testing.T) { } count := page["count"] - if count != 1 { + if count != float64(1) { t.Errorf("expected page count to be 1, got %v", count) } } @@ -867,6 +867,11 @@ func TestNuGetCooldownFilteringWithPackageOverride(t *testing.T) { }, } + body, err := json.Marshal(registration) + if err != nil { + t.Fatal(err) + } + proxy := testProxy() proxy.Cooldown = &cooldown.Config{ Default: "3d", @@ -878,11 +883,17 @@ func TestNuGetCooldownFilteringWithPackageOverride(t *testing.T) { proxyURL: "http://localhost:8080", } - if !h.filterNuGetRegistration(registration, "") { - t.Fatal("expected registration items to be retained") + filtered, err := h.applyCooldownFiltering(body) + if err != nil { + t.Fatal(err) } - pages := registration["items"].([]any) + var result map[string]any + if err := json.Unmarshal(filtered, &result); err != nil { + t.Fatal(err) + } + + pages := result["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) @@ -909,20 +920,36 @@ func TestNuGetCooldownNoCooldownConfig(t *testing.T) { }, } - // No cooldown: all registration items are retained. + body, err := json.Marshal(registration) + if err != nil { + t.Fatal(err) + } + + // No cooldown - applyCooldownFiltering still works, just doesn't filter h := &NuGetHandler{ proxy: testProxy(), proxyURL: "http://localhost:8080", } - if !h.filterNuGetRegistration(registration, "") { - t.Fatal("expected registration items to be retained") + filtered, err := h.applyCooldownFiltering(body) + if err != nil { + t.Fatal(err) } - pages := registration["items"].([]any) + var result map[string]any + if err := json.Unmarshal(filtered, &result); err != nil { + t.Fatal(err) + } + + pages := result["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) + // Without cooldown config on the handler, applyCooldownFiltering + // is called but proxy.Cooldown is nil, so IsAllowed is never called + // Actually, applyCooldownFiltering always runs the filter logic - + // but the caller (handleRegistration) short-circuits when cooldown is disabled. + // The function itself should still work fine with a nil Cooldown. if len(items) != 1 { t.Fatalf("expected 1 item, got %d", len(items)) } @@ -951,6 +978,11 @@ func TestNuGetCooldownFilteringNuGetTimestamp(t *testing.T) { }, } + body, err := json.Marshal(registration) + if err != nil { + t.Fatal(err) + } + proxy := testProxy() proxy.Cooldown = &cooldown.Config{ Default: "3d", @@ -961,11 +993,17 @@ func TestNuGetCooldownFilteringNuGetTimestamp(t *testing.T) { proxyURL: "http://localhost:8080", } - if !h.filterNuGetRegistration(registration, "") { - t.Fatal("expected registration items to be retained") + filtered, err := h.applyCooldownFiltering(body) + if err != nil { + t.Fatal(err) } - pages := registration["items"].([]any) + var result map[string]any + if err := json.Unmarshal(filtered, &result); err != nil { + t.Fatal(err) + } + + pages := result["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) @@ -1064,29 +1102,3 @@ func TestNuGetHandleRegistrationWithoutCooldown(t *testing.T) { t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } } - -func TestNuGetRetentionKey(t *testing.T) { - tests := []struct { - key string - want string - ok bool - }{ - {"pkg:nuget/Newtonsoft.Json", "pkg:nuget/newtonsoft.json", true}, - {"pkg:nuget/serilog", "pkg:nuget/serilog", true}, - {"pkg:nuget/serilog/serilog", "", false}, - {"pkg:npm/serilog", "", false}, - } - for _, tt := range tests { - p, err := purl.Parse(tt.key) - if err != nil { - t.Fatalf("Parse(%s): %v", tt.key, err) - } - got, ok := nugetRetentionKey(p) - if ok != tt.ok || (ok && got != tt.want) { - t.Errorf("nugetRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) - } - } - if _, ok := nugetRetentionKey(nil); ok { - t.Error("nugetRetentionKey(nil) accepted") - } -} diff --git a/internal/handler/package_routes.go b/internal/handler/package_routes.go deleted file mode 100644 index c56c3fb..0000000 --- a/internal/handler/package_routes.go +++ /dev/null @@ -1,92 +0,0 @@ -package handler - -import ( - "crypto/sha256" - "encoding/hex" - "path" - "sort" - "strings" -) - -// routeIDLength is the number of hex characters of the upstream URL digest -// that qualify cache entries of a routed package. -const routeIDLength = 16 - -// packageRoutes sends packages whose names match a pattern to a dedicated -// upstream instead of the ecosystem's default registry. A matched package is -// never looked up anywhere else, so a public package with the same name cannot -// replace it. -type packageRoutes struct { - routes []packageRoute // longest pattern first -} - -type packageRoute struct { - pattern string - url string - id string -} - -func newPackageRoutes(patterns map[string]string) packageRoutes { - routes := make([]packageRoute, 0, len(patterns)) - for pattern, upstreamURL := range patterns { - upstreamURL = strings.TrimSuffix(upstreamURL, "/") - routes = append(routes, packageRoute{ - pattern: strings.ToLower(pattern), - url: upstreamURL, - id: routeID(upstreamURL), - }) - } - sort.Slice(routes, func(i, j int) bool { - if len(routes[i].pattern) != len(routes[j].pattern) { - return len(routes[i].pattern) > len(routes[j].pattern) - } - return routes[i].pattern < routes[j].pattern - }) - return packageRoutes{routes: routes} -} - -// routeID identifies a route's upstream in cache keys and filenames. -func routeID(upstreamURL string) string { - digest := sha256.Sum256([]byte(strings.TrimSuffix(upstreamURL, "/"))) - return hex.EncodeToString(digest[:])[:routeIDLength] -} - -// RoutedFilenamePrefix returns what the npm and Composer handlers put in -// front of the artifact filenames of a package routed to routeURL. -func RoutedFilenamePrefix(routeURL string) string { - return routeID(routeURL) + "/" -} - -// match returns the route for a package name, if any pattern matches it. -func (r packageRoutes) match(packageName string) (packageRoute, bool) { - name := strings.ToLower(packageName) - for _, route := range r.routes { - if ok, _ := path.Match(route.pattern, name); ok { - return route, true - } - } - return packageRoute{}, false -} - -// cacheKey qualifies a package's metadata cache key with the route's upstream, -// so entries cached from the default registry, or from a route's previous -// upstream, are never served for a routed package. -func (r packageRoute) cacheKey(packageName string) string { - return "routes/" + r.id + "/" + packageName -} - -// cacheFilename qualifies a routed package's artifact filename the same way as -// cacheKey, and leaves other packages' filenames unchanged. -func (r packageRoutes) cacheFilename(packageName, filename string) string { - if route, routed := r.match(packageName); routed { - return route.id + "/" + filename - } - return filename -} - -// RoutedPackageURL returns the upstream URL a route in routes sends -// packageName to, as the npm and Composer handlers pick it. -func RoutedPackageURL(routes map[string]string, packageName string) (string, bool) { - route, ok := newPackageRoutes(routes).match(packageName) - return route.url, ok -} diff --git a/internal/handler/package_routes_test.go b/internal/handler/package_routes_test.go deleted file mode 100644 index dcb6f5f..0000000 --- a/internal/handler/package_routes_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package handler - -import "testing" - -func TestPackageRoutesMatch(t *testing.T) { - routes := newPackageRoutes(map[string]string{ - "@example/*": "https://npm.example.com/scoped/", - "@example/tool-*": "https://npm.example.com/tools", - "example/*": "https://composer.example.com/packages", - "single-package": "https://registry.example.com/single", - }) - - tests := []struct { - name string - pkg string - wantURL string - wantOK bool - }{ - {"scoped npm package", "@example/widgets", "https://npm.example.com/scoped", true}, - {"longest pattern wins", "@example/tool-kit", "https://npm.example.com/tools", true}, - {"composer vendor", "example/library", "https://composer.example.com/packages", true}, - {"exact name", "single-package", "https://registry.example.com/single", true}, - {"names are case-insensitive", "Example/Library", "https://composer.example.com/packages", true}, - {"wildcard does not cross slash", "example/library/extra", "", false}, - {"similar vendor does not match", "example-fork/library", "", false}, - {"unrouted package", "lodash", "", false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - route, ok := routes.match(tt.pkg) - if ok != tt.wantOK { - t.Fatalf("match(%q) ok = %t, want %t", tt.pkg, ok, tt.wantOK) - } - if ok && route.url != tt.wantURL { - t.Errorf("match(%q) url = %q, want %q", tt.pkg, route.url, tt.wantURL) - } - }) - } -} - -func TestPackageRoutesCacheKeysAreDistinct(t *testing.T) { - routes := newPackageRoutes(map[string]string{ - "example/*": "https://one.example.com", - "other-example/*": "https://two.example.com", - }) - - first, _ := routes.match("example/library") - second, _ := routes.match("other-example/library") - - if first.cacheKey("example/library") == "example/library" { - t.Error("routed cache key must differ from the unrouted package name") - } - if first.cacheKey("example/library") == second.cacheKey("example/library") { - t.Error("routes with different upstreams must not share cache keys") - } - if containsPathTraversal(first.cacheKey("example/library")) { - t.Error("routed cache key must be usable as a storage path") - } -} diff --git a/internal/handler/profile.go b/internal/handler/profile.go deleted file mode 100644 index 233e7bb..0000000 --- a/internal/handler/profile.go +++ /dev/null @@ -1,30 +0,0 @@ -package handler - -import "time" - -// WithCooldown creates a policy view sharing storage, fetches and rewrite space. -// Configure the original proxy before constructing its views. -func (p *Proxy) WithCooldown(policy CooldownPolicy, shortest time.Duration) *Proxy { - return &Proxy{ - DB: p.DB, Storage: p.Storage, Fetcher: p.Fetcher, Resolver: p.Resolver, - RouteFor: p.RouteFor, Logger: p.Logger, - Cooldown: policy, ShortestCooldown: shortest, Denylist: p.Denylist, - CacheMetadata: p.CacheMetadata, MetadataTTL: p.MetadataTTL, - MetadataMaxSize: p.MetadataMaxSize, MetadataStreamMaxSize: p.MetadataStreamMaxSize, - GradleReadOnly: p.GradleReadOnly, HostedEnabled: p.HostedEnabled, - GradleMaxUploadSize: p.GradleMaxUploadSize, NPMFullMetadata: p.NPMFullMetadata, - DirectServe: p.DirectServe, DirectServeTTL: p.DirectServeTTL, - DirectServeBaseURL: p.DirectServeBaseURL, HTTPClient: p.HTTPClient, - RateLimits: p.RateLimits, HostedNPMScopes: p.HostedNPMScopes, - HostedComposerVendors: p.HostedComposerVendors, StreamArtifacts: p.StreamArtifacts, - Scanners: p.Scanners, ScanSigningKey: p.ScanSigningKey, ScanFetchBaseURL: p.ScanFetchBaseURL, - rewrites: p.rewrites, now: p.now, started: p.started, shared: p.sharedProxy(), - } -} - -func (p *Proxy) sharedProxy() *Proxy { - if p.shared != nil { - return p.shared - } - return p -} diff --git a/internal/handler/profile_test.go b/internal/handler/profile_test.go deleted file mode 100644 index 5caf7b8..0000000 --- a/internal/handler/profile_test.go +++ /dev/null @@ -1,234 +0,0 @@ -package handler - -import ( - "context" - "errors" - "net/http" - "net/http/httptest" - "reflect" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/proxy/internal/denylist" - "github.com/git-pkgs/proxy/internal/scanner" - registryclient "github.com/git-pkgs/registries/client" -) - -func TestWithCooldownCopiesProxyFields(t *testing.T) { - p, _, _, _ := setupTestProxy(t) - p.RouteFor = func(string, string) *ArtifactRoute { return nil } - p.Cooldown = profilePolicy(t, "1d") - p.ShortestCooldown = 24 * time.Hour - p.Denylist = &denylist.Policy{} - p.CacheMetadata = true - p.MetadataTTL = time.Hour - p.MetadataMaxSize = 1024 - p.MetadataStreamMaxSize = 2048 - p.GradleReadOnly = true - p.HostedEnabled = true - p.GradleMaxUploadSize = 4096 - p.NPMFullMetadata = true - p.DirectServe = true - p.DirectServeTTL = time.Minute - p.DirectServeBaseURL = "https://storage.example.com" - p.RateLimits = registryclient.NewRateLimitGate(http.DefaultTransport) - p.HostedNPMScopes = []string{"@example"} - p.HostedComposerVendors = []string{"example"} - p.StreamArtifacts = true - p.Scanners = &scanner.Group{} - p.ScanSigningKey = []byte("signing-key") - p.ScanFetchBaseURL = "http://scanner.example.com" - p.SetMetadataRewriteCacheSize(1024) - p.inFlight = map[string]*inflightFetch{"artifact": {}} - p.detachSlots = make(chan struct{}, 1) - p.inFlightMeta = map[string]*inflightMetadata{"metadata": {}} - p.publishIndexes.entries = map[string]publishIndexEntry{"document": {}} - p.cooldownWatch.active = map[*cooldownRecorder]struct{}{{}: {}} - - policy := profilePolicy(t, "3d") - view := p.WithCooldown(policy, 72*time.Hour) - if view.Cooldown != policy || view.ShortestCooldown != 72*time.Hour || view.shared != p { - t.Fatal("view did not select its policy and shared proxy") - } - original, copied := reflect.ValueOf(p).Elem(), reflect.ValueOf(view).Elem() - for i := range original.NumField() { - name := original.Type().Field(i).Name - before, after := original.Field(i), copied.Field(i) - switch name { - case "Cooldown", "ShortestCooldown", "shared": - continue - case "fetchMu", "inFlight", "detachSlots", "metaMu", "inFlightMeta", "publishIndexes", "cooldownWatch": - if !after.IsZero() { - t.Errorf("view field %s is not empty", name) - } - continue - } - if before.IsZero() { - t.Errorf("populate Proxy.%s to check whether WithCooldown copies it", name) - continue - } - switch before.Kind() { - case reflect.Func, reflect.Slice: - if before.Pointer() != after.Pointer() { - t.Errorf("WithCooldown did not copy %s", name) - } - if before.Kind() == reflect.Slice && (before.Len() != after.Len() || before.Cap() != after.Cap()) { - t.Errorf("WithCooldown changed slice bounds for %s", name) - } - default: - if !before.Equal(after) { - t.Errorf("WithCooldown did not copy %s", name) - } - } - } -} - -func TestProfilesShareDownloadPublishIndexes(t *testing.T) { - for _, c := range downloadCooldownCases() { - if c.ecosystem != "conda" && c.ecosystem != "helm" { - continue - } - t.Run(c.ecosystem, func(t *testing.T) { - var calls atomic.Int64 - times := map[string]time.Time{cooldownRecentVersion: time.Now().Add(-4 * 24 * time.Hour)} - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) - c.metadata(w, r, times) - })) - t.Cleanup(upstream.Close) - p, _, _, fetcher := setupTestProxy(t) - p.HTTPClient = upstream.Client() - p.Cooldown = profilePolicy(t, "1d") - fetcher.fetchErr = errors.New("artifact must be served from the cache") - c.seedCachedArtifact(t, p, cooldownRecentVersion) - permissive := p.WithCooldown(profilePolicy(t, "3d"), 72*time.Hour) - strict := p.WithCooldown(profilePolicy(t, "7d"), 168*time.Hour) - nested := permissive.WithCooldown(profilePolicy(t, "5d"), 120*time.Hour) - for _, request := range []struct { - proxy *Proxy - want int - }{ - {permissive, http.StatusOK}, - {strict, http.StatusNotFound}, - {p, http.StatusOK}, - {nested, http.StatusNotFound}, - } { - w := serveGet(t, c.handler(request.proxy, upstream.URL), c.path(cooldownRecentVersion)) - wantStatus(t, w, request.want, "") - } - if n := calls.Load(); n != 1 { - t.Errorf("index downloads across profiles = %d, want 1", n) - } - if n := len(p.publishIndexes.entries); n != 1 { - t.Errorf("shared publish indexes = %d, want 1", n) - } - if fetcher.fetchCalled { - t.Error("cached artifact was fetched again") - } - }) - } -} - -func profilePolicy(t *testing.T, duration string) *cooldown.Policy { - t.Helper() - policy, err := cooldown.NewPolicy(&cooldown.Config{Default: duration}, nil) - if err != nil { - t.Fatal(err) - } - return policy -} - -func TestProfilesCoalesceUncachedMetadata(t *testing.T) { - u := newMetadataUpstream(t, http.StatusOK) - p := metadataTestProxy(t, u) - views := []*Proxy{p.WithCooldown(profilePolicy(t, "3d"), 72*time.Hour), p.WithCooldown(profilePolicy(t, "7d"), 168*time.Hour)} - first := startMetadataCalls(views[0], context.Background(), 1, u.URL, contentTypeJSON) - <-u.entered - second := startMetadataCalls(views[1], context.Background(), 1, u.URL, contentTypeJSON) - key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) - var release sync.Once - t.Cleanup(func() { release.Do(func() { close(u.release) }) }) - waitForMetadataWaiters(t, p, key, 1) - release.Do(func() { close(u.release) }) - for _, calls := range [][]chan metadataCall{first, second} { - result := <-calls[0] - if result.err != nil || !strings.Contains(string(result.body), "accept") { - t.Fatalf("metadata = %s, error %v", result.body, result.err) - } - } - if n := u.calls.Load(); n != 1 { - t.Fatalf("upstream requests = %d, want 1", n) - } -} - -func TestProfilesCoalesceArtifactRequests(t *testing.T) { - proxy, db, _, _ := setupTestProxy(t) - if err := db.SetVersionPublishedAt("pkg:cargo/demo@1.0.0", "pkg:cargo/demo", time.Now().Add(-10*24*time.Hour)); err != nil { - t.Fatal(err) - } - fetcher := &countingFetcher{content: "crate bytes", delay: fetchHoldTime} - proxy.Fetcher = fetcher - views := []*Proxy{proxy.WithCooldown(profilePolicy(t, "3d"), 72*time.Hour), proxy.WithCooldown(profilePolicy(t, "7d"), 168*time.Hour)} - var wg sync.WaitGroup - start := make(chan struct{}) - for range 4 { - for _, view := range views { - wg.Go(func() { - <-start - h := NewCargoHandler(view, "http://proxy.test", "https://index.crates.io", "https://static.crates.io/crates") - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/crates/demo/1.0.0/download", nil)) - if w.Code != http.StatusOK || w.Body.String() != "crate bytes" { - t.Errorf("artifact response %d: %s", w.Code, w.Body) - } - }) - } - } - close(start) - wg.Wait() - if n := fetcher.calls.Load(); n != 1 { - t.Fatalf("upstream fetches = %d, want 1", n) - } -} - -type profileClockPolicy struct { - *cooldown.Policy - now func() time.Time -} - -func (p profileClockPolicy) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { - return p.Evaluate(ecosystem, packagePURL, publishedAt, p.now()).Allowed -} - -func TestProfileExpiryReleasesVersionsIndependently(t *testing.T) { - u, proxy, _ := gemCacheSetup(t) - clock := &testClock{t: time.Now()} - proxy.now = clock.now - proxy.SetMetadataRewriteCacheSize(1 << 20) - view := proxy.WithCooldown(profileClockPolicy{profilePolicy(t, "3d"), clock.now}, 72*time.Hour) - h := NewGemHandlerWithUpstream(view, "http://proxy.test/cooldown/3d", u.URL).Routes() - strictView := proxy.WithCooldown(profileClockPolicy{profilePolicy(t, "7d"), clock.now}, 168*time.Hour) - strict := NewGemHandlerWithUpstream(strictView, "http://proxy.test/cooldown/7d", u.URL).Routes() - first := getWith(t, h, gemInfoPath, nil) - if first.Code != http.StatusOK || first.Body.String() != gemFiltered { - t.Fatalf("initial response %d: %s", first.Code, first.Body) - } - if first := getWith(t, strict, gemInfoPath, nil); first.Code != http.StatusOK || first.Body.String() != gemFiltered { - t.Fatalf("initial strict response %d: %s", first.Code, first.Body) - } - clock.advance(4 * 24 * time.Hour) - second := getWith(t, h, gemInfoPath, nil) - if second.Code != http.StatusOK || !strings.Contains(second.Body.String(), "2.0.0 |") { - t.Fatalf("version did not leave cooldown: %d %s", second.Code, second.Body) - } - if second := getWith(t, strict, gemInfoPath, nil); second.Code != http.StatusOK || second.Body.String() != gemFiltered { - t.Fatalf("strict profile released version early: %d %s", second.Code, second.Body) - } - if n, _ := u.counts(gemInfoPath); n != 1 { - t.Errorf("expiry refetched raw metadata %d times", n) - } -} diff --git a/internal/handler/proxy_cached_encoding_test.go b/internal/handler/proxy_cached_encoding_test.go deleted file mode 100644 index 45eaef3..0000000 --- a/internal/handler/proxy_cached_encoding_test.go +++ /dev/null @@ -1,260 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "errors" - "net/http" - "net/http/httptest" - "strconv" - "strings" - "sync/atomic" - "testing" - "time" -) - -// gzipWhenAskedUpstream serves compressed bytes with Content-Encoding: gzip -// when the request advertises gzip, plain bytes otherwise, like a CDN that -// compresses on the fly. It records the last Accept-Encoding it saw and counts -// every request before the availability gate so a cache-miss refetch during a -// simulated outage is observable. -type gzipWhenAskedUpstream struct { - *httptest.Server - available atomic.Bool - requests atomic.Int32 - acceptEncoding atomic.Value // string -} - -func newGzipWhenAskedUpstream(plain, compressed []byte) *gzipWhenAskedUpstream { - u := &gzipWhenAskedUpstream{} - u.available.Store(true) - u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - u.requests.Add(1) - u.acceptEncoding.Store(r.Header.Get(headerAcceptEncoding)) - if !u.available.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set(headerContentType, contentTypeJSON) - if strings.Contains(r.Header.Get(headerAcceptEncoding), "gzip") { - w.Header().Set(headerContentEncoding, "gzip") - _, _ = w.Write(compressed) - return - } - _, _ = w.Write(plain) - })) - return u -} - -func (u *gzipWhenAskedUpstream) sawAcceptEncoding() string { - s, _ := u.acceptEncoding.Load().(string) - return s -} - -// serveGzip issues one request through proxyCachedWithEncoding asking the -// upstream for gzip. -func serveGzip(proxy *Proxy, upstreamURL string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodGet, "/index.json", nil) - proxy.proxyCachedWithEncoding(w, r, upstreamURL, "gzip-test", "index", "gzip", nil, "*/*") - return w -} - -func assertGzipResponse(t *testing.T, label string, w *httptest.ResponseRecorder, compressed []byte) { - t.Helper() - if w.Code != http.StatusOK { - t.Fatalf("%s: status = %d, want 200: %s", label, w.Code, w.Body.String()) - } - if !bytes.Equal(w.Body.Bytes(), compressed) { - t.Errorf("%s: body is not the compressed bytes (got %d, want %d)", label, w.Body.Len(), len(compressed)) - } - if got := w.Header().Get(headerContentEncoding); got != "gzip" { - t.Errorf("%s: Content-Encoding = %q, want %q", label, got, "gzip") - } - if got := w.Header().Get(headerContentLength); got != strconv.Itoa(len(compressed)) { - t.Errorf("%s: Content-Length = %q, want %d", label, got, len(compressed)) - } -} - -// TestProxyCachedWithEncoding_GzipCachesAndReplays covers the cached path: -// requesting gzip upstream stores the compressed bytes plus Content-Encoding -// and replays both from cache without contacting the upstream again. -func TestProxyCachedWithEncoding_GzipCachesAndReplays(t *testing.T) { - plain := []byte(`{"packages":{}}`) - compressed := gzipPayload(t, plain) - upstream := newGzipWhenAskedUpstream(plain, compressed) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - - first := serveGzip(proxy, upstream.URL+"/index.json") - assertGzipResponse(t, "first", first, compressed) - if got := upstream.sawAcceptEncoding(); got != "gzip" { - t.Errorf("upstream Accept-Encoding = %q, want %q", got, "gzip") - } - - before := upstream.requests.Load() - upstream.available.Store(false) - cached := serveGzip(proxy, upstream.URL+"/index.json") - assertGzipResponse(t, "cached", cached, compressed) - if upstream.requests.Load() != before { - t.Errorf("cached replay hit upstream: requests %d -> %d", before, upstream.requests.Load()) - } -} - -// TestProxyCachedWithEncoding_GzipStreamPath covers the cache_metadata=false -// branch: the streaming path must request gzip and forward Content-Encoding. -func TestProxyCachedWithEncoding_GzipStreamPath(t *testing.T) { - plain := []byte(`{"packages":{}}`) - compressed := gzipPayload(t, plain) - upstream := newGzipWhenAskedUpstream(plain, compressed) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = false - proxy.HTTPClient = upstream.Client() - - w := serveGzip(proxy, upstream.URL+"/index.json") - assertGzipResponse(t, "stream", w, compressed) - if got := upstream.sawAcceptEncoding(); got != "gzip" { - t.Errorf("stream path upstream Accept-Encoding = %q, want %q", got, "gzip") - } -} - -// TestProxyCachedWithEncoding_GzipSurvivesCacheWriteFailure covers the failure -// the gzip mode makes reachable: when the metadata cache write fails the -// freshly fetched body is still served, so its Content-Encoding must come from -// the fetch and not from the (unwritten) cache row -- otherwise gzip bytes go -// out labelled application/json with no Content-Encoding. -func TestProxyCachedWithEncoding_GzipSurvivesCacheWriteFailure(t *testing.T) { - plain := []byte(`{"packages":{}}`) - compressed := gzipPayload(t, plain) - upstream := newGzipWhenAskedUpstream(plain, compressed) - defer upstream.Close() - - proxy, _, store, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.HTTPClient = upstream.Client() - store.storeErr = errors.New("disk full") - - w := serveGzip(proxy, upstream.URL+"/index.json") - assertGzipResponse(t, "store-failure", w, compressed) -} - -// TestProxyCachedWithEncoding_GzipStaleFallbackKeepsEncoding pins the -// stale-fallback return: when the upstream fails after the entry has expired, -// the stored gzip blob is served with its Content-Encoding taken from the -// cache row. -func TestProxyCachedWithEncoding_GzipStaleFallbackKeepsEncoding(t *testing.T) { - plain := []byte(`{"packages":{}}`) - compressed := gzipPayload(t, plain) - upstream := newGzipWhenAskedUpstream(plain, compressed) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = 0 // every request revalidates; an upstream failure falls back to the stale row - proxy.HTTPClient = upstream.Client() - - first := serveGzip(proxy, upstream.URL+"/index.json") - assertGzipResponse(t, "first", first, compressed) - - upstream.available.Store(false) - stale := serveGzip(proxy, upstream.URL+"/index.json") - assertGzipResponse(t, "stale", stale, compressed) -} - -// TestProxyCachedWithEncoding_UpsertFailureDiscardsBlob covers the row-write -// failure: when the gzip blob is stored but the cache row cannot be updated, -// the blob must be discarded so a later stale fallback cannot serve gzip -// bytes with the previous row's encoding. The fresh response is still -// correct because its encoding comes from the fetch. -func TestProxyCachedWithEncoding_UpsertFailureDiscardsBlob(t *testing.T) { - plain := []byte(`{"packages":{}}`) - compressed := gzipPayload(t, plain) - upstream := newGzipWhenAskedUpstream(plain, compressed) - defer upstream.Close() - - proxy, db, store, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = 0 // every request revalidates - proxy.HTTPClient = upstream.Client() - - // Seed an identity row + plain blob, as every key has before the gzip rollout. - w := httptest.NewRecorder() - proxy.proxyCachedWithEncoding(w, httptest.NewRequest(http.MethodGet, "/index.json", nil), - upstream.URL+"/index.json", "gzip-test", "index", "identity", nil, "*/*") - if w.Code != http.StatusOK { - t.Fatalf("seed status = %d, want 200", w.Code) - } - - // Now DB writes fail while reads keep working. - db.SetMaxOpenConns(1) - if _, err := db.Exec("PRAGMA query_only=1"); err != nil { - t.Fatalf("PRAGMA query_only=1: %v", err) - } - fresh := serveGzip(proxy, upstream.URL+"/index.json") - assertGzipResponse(t, "fresh with failed row write", fresh, compressed) - - storagePath := metadataStoragePath("gzip-test", "index") - if exists, _ := store.Exists(context.Background(), storagePath); exists { - t.Fatalf("blob %s still present after the row write failed", storagePath) - } - - // Upstream down: the stale fallback must not serve the orphaned gzip - // blob under the old identity row. - if _, err := db.Exec("PRAGMA query_only=0"); err != nil { - t.Fatalf("PRAGMA query_only=0: %v", err) - } - upstream.available.Store(false) - stale := serveGzip(proxy, upstream.URL+"/index.json") - if stale.Code == http.StatusOK { - t.Fatalf("stale fallback served status 200 (Content-Encoding=%q, %d bytes) from an orphaned blob; want an error", - stale.Header().Get(headerContentEncoding), stale.Body.Len()) - } -} - -// TestProxyCachedWithEncoding_StaleFallbackRereadsRow covers the rollout race: -// a request that read the identity row, lost the upstream race to a request -// that stored the gzip blob, and then failed upstream must label the blob -// with the row as it is now, not with the row it read at the start. -func TestProxyCachedWithEncoding_StaleFallbackRereadsRow(t *testing.T) { - plain := []byte(`{"packages":{}}`) - compressed := gzipPayload(t, plain) - - var proxy *Proxy - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if requests.Add(1) == 1 { - w.Header().Set(headerContentType, contentTypeJSON) - _, _ = w.Write(plain) // seed request: identity - return - } - // Second request has already read the identity row. Simulate a - // concurrent request finishing first: store the gzip blob and row, - // then fail this request so it takes the stale fallback. - proxy.cacheMetadataBlob(r.Context(), "gzip-test", "index", metadataStoragePath("gzip-test", "index"), - &upstreamMetadata{body: compressed, contentType: contentTypeJSON, contentEncoding: "gzip"}) - http.Error(w, "unavailable", http.StatusServiceUnavailable) - })) - defer upstream.Close() - - proxy, _, _, _ = setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = 0 - proxy.HTTPClient = upstream.Client() - - w := httptest.NewRecorder() - proxy.proxyCachedWithEncoding(w, httptest.NewRequest(http.MethodGet, "/index.json", nil), - upstream.URL+"/index.json", "gzip-test", "index", "identity", nil, "*/*") - if w.Code != http.StatusOK { - t.Fatalf("seed status = %d, want 200", w.Code) - } - - raced := serveGzip(proxy, upstream.URL+"/index.json") - assertGzipResponse(t, "stale fallback after concurrent gzip store", raced, compressed) -} diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 5a6d426..60bbbad 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -5,32 +5,17 @@ import ( "errors" "fmt" "net/http" - "net/url" - "regexp" "strings" "time" - "github.com/Masterminds/semver/v3" - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( - pubUpstream = "https://pub.dev" - pubPathParts = 2 // name + version in path split by /versions/ - pubArchiveExt = ".tar.gz" + pubUpstream = "https://pub.dev" + pubPathParts = 2 // name + version in path split by /versions/ ) -// pubPackageName matches pub package names, which have to be Dart identifiers. -var pubPackageName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) - -// Packages are stored under pkg:pub/{name} with the name from the download -// path, and purl keeps pub names as they are, so package rules use the -// default key mapping. -func init() { - retention.Default.Register(retention.Spec{Key: "pub", CanonicalPackage: retention.DefaultCanonical("pub")}) -} - // PubHandler handles pub.dev registry protocol requests. type PubHandler struct { proxy *Proxy @@ -47,13 +32,6 @@ func NewPubHandler(proxy *Proxy, proxyURL string) *PubHandler { } } -// NewPubHandlerWithUpstream creates a pub handler with a custom upstream. -func NewPubHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *PubHandler { - h := NewPubHandler(proxy, proxyURL) - h.upstreamURL = configuredUpstreamURL(upstreamURL, pubUpstream) - return h -} - // Routes returns the HTTP handler for pub requests. func (h *PubHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -77,99 +55,26 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } name := parts[0] - // Only the archive is cached. Without the suffix the upstream path is the - // HTML version page on pub.dev, which must not land under the archive key. - version, ok := strings.CutSuffix(parts[1], pubArchiveExt) - if !ok { - http.Error(w, "not found", http.StatusNotFound) - return - } + version := strings.TrimSuffix(parts[1], ".tar.gz") - if !pubPackageName.MatchString(name) || version == "" || strings.Contains(version, "/") { + if name == "" || version == "" { http.Error(w, "invalid request", http.StatusBadRequest) return } - filename := name + "-" + version + pubArchiveExt + filename := fmt.Sprintf("%s-%s.tar.gz", name, version) - h.proxy.Logger.Debug("pub download request", + h.proxy.Logger.Info("pub download request", "name", name, "version", version) - h.discardVersionPage(name, version, filename) - if h.proxy.withholdInCooldown(w, r, downloadVersion{ - ecosystem: "pub", policyName: name, storedName: name, version: version, cacheName: name, filename: filename, - lookup: func() (time.Time, bool, bool, error) { - published, ok, err := h.publishTime(r, name, version) - return published, ok, false, err - }, - }) { + result, err := h.proxy.GetOrFetchArtifact(r.Context(), "pub", name, version, filename) + if err != nil { + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch package", http.StatusBadGateway) return } - downloadURL := fmt.Sprintf("%s/packages/%s/versions/%s%s", - h.upstreamURL, url.PathEscape(name), url.PathEscape(version), pubArchiveExt) - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), "pub", name, version, filename, downloadURL, - ) - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") - return - } - - ServeArtifactRequest(w, r, result) -} - -// discardVersionPage clears a cached archive that was fetched from a path -// without the archive suffix. Proxy versions before the suffix check cached -// pub.dev's HTML version page under the archive's name that way. -func (h *PubHandler) discardVersionPage(name, version, filename string) { - _, versionPURL, err := packagePURLStrings("pub", name, version) - if err != nil { - return - } - record, err := h.proxy.DB.GetArtifact(versionPURL, filename) - if err != nil { - h.proxy.Logger.Warn("failed to read pub cache record", - "purl", versionPURL, "filename", filename, "error", err) - return - } - if record == nil || !record.StoragePath.Valid || record.Source == database.SourceLocal || - strings.HasSuffix(record.UpstreamURL, pubArchiveExt) { - return - } - h.proxy.Logger.Warn("cached pub archive was fetched without the archive suffix, discarding", - "purl", versionPURL, "filename", filename, "upstream_url", record.UpstreamURL) - if err := h.proxy.DB.DiscardArtifact(versionPURL, filename, record.StoragePath.String); err != nil { - h.proxy.Logger.Warn("failed to discard pub cache record", - "purl", versionPURL, "filename", filename, "error", err) - } -} - -// publishTime reads a version's published time from the package metadata, -// sharing the metadata cache with handlePackageMetadata. -func (h *PubHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { - upstreamURL := fmt.Sprintf("%s/api/packages/%s", h.upstreamURL, name) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pub", name, upstreamURL) - if err != nil { - return time.Time{}, false, err - } - var metadata struct { - Versions []struct { - Version string `json:"version"` - Published string `json:"published"` - } `json:"versions"` - } - if err := json.Unmarshal(body, &metadata); err != nil { - return time.Time{}, false, err - } - for _, v := range metadata.Versions { - if v.Version != version { - continue - } - published, err := time.Parse(time.RFC3339, v.Published) - return published, err == nil, nil - } - return time.Time{}, false, nil + ServeArtifact(w, result) } // handlePackageMetadata proxies package metadata and rewrites archive URLs. @@ -180,7 +85,7 @@ func (h *PubHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - h.proxy.Logger.Debug("pub metadata request", "package", name) + h.proxy.Logger.Info("pub metadata request", "package", name) upstreamURL := fmt.Sprintf("%s/api/packages/%s", h.upstreamURL, name) @@ -190,9 +95,6 @@ func (h *PubHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques http.Error(w, "not found", http.StatusNotFound) return } - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return @@ -200,19 +102,14 @@ func (h *PubHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques rewritten, err := h.rewriteMetadata(name, body) if err != nil { - if h.proxy.mustFilterMetadata("pub", name) { - h.proxy.Logger.Error("failed to filter pub metadata", "package", name, "error", err) - http.Error(w, "failed to filter package metadata", http.StatusBadGateway) - return - } h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) - w.Header().Set(headerContentType, "application/json") + w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write(body) return } - w.Header().Set(headerContentType, "application/json") + w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write(rewritten) } @@ -230,7 +127,7 @@ func (h *PubHandler) rewriteMetadata(name string, body []byte) ([]byte, error) { return body, nil } - packagePURL := canonicalPackagePURL("pub", name) + packagePURL := purl.MakePURLString("pub", name, "") filtered := h.filterAndRewriteVersions(name, packagePURL, versions) metadata["versions"] = filtered @@ -285,7 +182,7 @@ func (h *PubHandler) shouldFilterVersion(packagePURL, name, version string, vmap return false } - if !h.proxy.cooldown().IsAllowed("pub", packagePURL, publishedAt) { + if !h.proxy.Cooldown.IsAllowed("pub", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering pub version", "package", name, "version", version) return true @@ -294,47 +191,32 @@ func (h *PubHandler) shouldFilterVersion(packagePURL, name, version string, vmap return false } -// updateLatestVersion points latest at the kept entry for its version, so -// its archive_url is the proxy's too. When cooldown removed that version, -// latest becomes the newest stable version left, or the newest version left -// if all are prereleases, and is dropped when none is left. +// updateLatestVersion updates the latest field if the current latest version +// was removed by cooldown filtering. func (h *PubHandler) updateLatestVersion(metadata map[string]any, filtered []any) { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return + } + latest, ok := metadata["latest"].(map[string]any) if !ok { return } - latestVer, _ := latest["version"].(string) - var newest, newestStable *semver.Version - var newestEntry, newestStableEntry any + latestVer, ok := latest["version"].(string) + if !ok { + return + } + for _, vdata := range filtered { - vmap, ok := vdata.(map[string]any) - if !ok { - continue - } - version, _ := vmap["version"].(string) - if version == latestVer { - latest["archive_url"] = vmap["archive_url"] - return - } - parsed, err := semver.NewVersion(version) - if err != nil { - continue - } - if newest == nil || parsed.GreaterThan(newest) { - newest, newestEntry = parsed, vdata - } - if parsed.Prerelease() == "" && (newestStable == nil || parsed.GreaterThan(newestStable)) { - newestStable, newestStableEntry = parsed, vdata + if vmap, ok := vdata.(map[string]any); ok { + if vmap["version"] == latestVer { + return + } } } - switch { - case newestStableEntry != nil: - metadata["latest"] = newestStableEntry - case newestEntry != nil: - metadata["latest"] = newestEntry - default: - delete(metadata, "latest") + if len(filtered) > 0 { + metadata["latest"] = filtered[len(filtered)-1] } } diff --git a/internal/handler/pub_test.go b/internal/handler/pub_test.go index 5311e25..8a4c098 100644 --- a/internal/handler/pub_test.go +++ b/internal/handler/pub_test.go @@ -2,16 +2,11 @@ package handler import ( "encoding/json" - "io" "log/slog" - "net/http" - "net/http/httptest" - "strings" "testing" "time" "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/registries/fetch" ) func TestPubRewriteMetadata(t *testing.T) { @@ -97,167 +92,3 @@ func TestPubRewriteMetadataCooldown(t *testing.T) { t.Errorf("latest version = %v, want 1.0.0", latest["version"]) } } - -// Without the .tar.gz suffix pub.dev answers with the HTML version page, which -// would otherwise be cached under the archive's key. -func TestPubDownloadRequiresArchiveSuffix(t *testing.T) { - proxy, db, _, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("version page")), - ContentType: "text/html", - } - h := NewPubHandlerWithUpstream(proxy, "http://localhost", "https://pub.example.com") - - req := httptest.NewRequest(http.MethodGet, "/packages/http/versions/1.2.0", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusNotFound { - t.Errorf("status = %d, want %d", w.Code, http.StatusNotFound) - } - if fetcher.fetchCalled { - t.Errorf("upstream fetched %q", fetcher.fetchedURL) - } - art, err := db.GetArtifact("pkg:pub/http@1.2.0", "http-1.2.0.tar.gz") - if err != nil { - t.Fatalf("GetArtifact: %v", err) - } - if art != nil { - t.Errorf("artifact cached from %q", art.UpstreamURL) - } -} - -func TestPubDownloadRejectsEncodedQueryInName(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("archive")), - ContentType: "application/octet-stream", - } - h := NewPubHandlerWithUpstream(proxy, "http://localhost", "https://pub.example.com") - - for _, path := range []string{ - "/packages/http%3Fx/versions/1.2.0.tar.gz", - "/packages/http%23x/versions/1.2.0.tar.gz", - "/packages/http/versions/1.2.0%3Fx.tar.gz", - } { - fetcher.fetchCalled = false - fetcher.fetchedURL = "" - req := httptest.NewRequest(http.MethodGet, path, nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if strings.ContainsAny(fetcher.fetchedURL, "?#") { - t.Errorf("%s: upstream URL %q carries a query or fragment", path, fetcher.fetchedURL) - } - if strings.Contains(path, "/http%") && fetcher.fetchCalled { - t.Errorf("%s: invalid package name reached upstream as %q", path, fetcher.fetchedURL) - } - } -} - -// Older versions cached the HTML version page as the archive. Such an entry -// is fetched again instead of being served. -func TestPubDownloadRefetchesCachedVersionPage(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - seedPackage(t, db, store, "pub", "http", "1.2.0", "http-1.2.0.tar.gz", "version page") - art, err := db.GetArtifact("pkg:pub/http@1.2.0", "http-1.2.0.tar.gz") - if err != nil || art == nil { - t.Fatalf("GetArtifact: %v", err) - } - art.UpstreamURL = "https://pub.dev/packages/http/versions/1.2.0" - if err := db.UpsertArtifact(art); err != nil { - t.Fatalf("UpsertArtifact: %v", err) - } - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("archive")), - ContentType: "application/octet-stream", - } - h := NewPubHandlerWithUpstream(proxy, "http://localhost", "https://pub.example.com") - - req := httptest.NewRequest(http.MethodGet, "/packages/http/versions/1.2.0.tar.gz", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if !fetcher.fetchCalled { - t.Fatal("cached version page was served without fetching upstream") - } - if got := w.Body.String(); got != "archive" { - t.Errorf("body = %q, want %q", got, "archive") - } -} - -func TestPubDownloadServesCachedArchive(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - seedPackage(t, db, store, "pub", "http", "1.2.0", "http-1.2.0.tar.gz", "cached archive") - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("refetched")), - ContentType: "application/octet-stream", - } - h := NewPubHandlerWithUpstream(proxy, "http://localhost", "https://pub.example.com") - - req := httptest.NewRequest(http.MethodGet, "/packages/http/versions/1.2.0.tar.gz", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if fetcher.fetchCalled { - t.Errorf("cached archive fetched again from %q", fetcher.fetchedURL) - } - if got := w.Body.String(); got != "cached archive" { - t.Errorf("body = %q, want %q", got, "cached archive") - } -} - -// TestPubRewriteMetadataLatest checks that latest points at the proxy and -// never names a version cooldown holds back, and that a held back latest is -// replaced with the newest stable version left. -func TestPubRewriteMetadataLatest(t *testing.T) { - old := time.Now().Add(-10 * 24 * time.Hour).Format(time.RFC3339) - recent := time.Now().Add(-time.Hour).Format(time.RFC3339) - entry := func(version, published string) string { - return `{"version":"` + version + `","published":"` + published + `","archive_url":"https://pub.dev/packages/demo/versions/` + version + `.tar.gz"}` - } - for _, tc := range []struct { - name, latest, versions string - cooldown bool - want, wantURL string - }{ - {"kept", entry("1.0.0", old), entry("1.0.0", old), false, "1.0.0", - "http://localhost:8080/pub/packages/demo/versions/1.0.0.tar.gz"}, - {"stable over prerelease", entry("2.0.0", recent), - entry("1.0.0", old) + "," + entry("1.1.0", old) + "," + entry("2.0.0-dev.1", old) + "," + entry("2.0.0", recent), true, "1.1.0", - "http://localhost:8080/pub/packages/demo/versions/1.1.0.tar.gz"}, - {"only prereleases", entry("2.0.0", recent), entry("2.0.0-dev.1", old) + "," + entry("2.0.0", recent), true, "2.0.0-dev.1", - "http://localhost:8080/pub/packages/demo/versions/2.0.0-dev.1.tar.gz"}, - {"all held back", entry("2.0.0", recent), entry("2.0.0", recent), true, "", ""}, - } { - t.Run(tc.name, func(t *testing.T) { - proxy := &Proxy{Logger: slog.Default()} - if tc.cooldown { - proxy.Cooldown = &cooldown.Config{Default: "3d"} - } - h := &PubHandler{proxy: proxy, proxyURL: "http://localhost:8080"} - output, err := h.rewriteMetadata("demo", []byte(`{"name":"demo","latest":`+tc.latest+`,"versions":[`+tc.versions+`]}`)) - if err != nil { - t.Fatal(err) - } - var result struct { - Latest *struct { - Version string `json:"version"` - ArchiveURL string `json:"archive_url"` - } `json:"latest"` - } - if err := json.Unmarshal(output, &result); err != nil { - t.Fatal(err) - } - if tc.want == "" { - if result.Latest != nil { - t.Errorf("latest = %+v, want none", *result.Latest) - } - return - } - if result.Latest == nil || result.Latest.Version != tc.want || result.Latest.ArchiveURL != tc.wantURL { - t.Errorf("latest = %+v, want %s at %s", result.Latest, tc.want, tc.wantURL) - } - }) - } -} diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 7db972e..3021d2b 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -1,82 +1,50 @@ package handler import ( - "bytes" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" - "maps" - "mime" "net/http" "net/url" "regexp" - "slices" - "strconv" "strings" "time" - "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( - pypiUpstream = "https://pypi.org" - pypiDownloadUpstream = "https://files.pythonhosted.org" - pypiSimpleJSON = "application/vnd.pypi.simple.v1+json" - pypiSimpleHTML = "application/vnd.pypi.simple.v1+html" - pypiSimpleLatestJSON = "application/vnd.pypi.simple.latest+json" - pypiSimpleLatestHTML = "application/vnd.pypi.simple.latest+html" - pypiLegacyHTML = "text/html" - pypiExactSpecificity = 2 - minWheelParts = 5 // name + version + python + abi + platform - minSubmatchParts = 2 // full match + first capture group - minPyPIPathParts = 3 // hash_prefix + hash + filename - minEggParts = 3 // name + version + python tag - - // PyPIMetadataSuffix is the PEP 658 core-metadata sidecar suffix that pip - // appends to a distribution URL when the index advertises core metadata. - // A sidecar resolves to the same name and version as the distribution it - // describes, so it is cached alongside it; consumers that expect an openable - // archive must skip these. - PyPIMetadataSuffix = ".metadata" + pypiUpstream = "https://pypi.org" + minWheelParts = 5 // name + version + python + abi + platform + minSubmatchParts = 2 // full match + first capture group + minPyPIPathParts = 3 // hash_prefix + hash + filename + minPythonTagLen = 2 // minimum length for a python tag (e.g., "py") ) // PyPIHandler handles PyPI registry protocol requests. type PyPIHandler struct { proxy *Proxy upstreamURL string - downloadURL string proxyURL string } -// PyPI files are stored under pkg:pypi/{name}, the name taken from the -// filename and normalized the way purl does it: lowercased, "_" turned into -// "-". Package rules use the default key mapping, so pkg:pypi/Typing_Extensions -// matches typing-extensions. Dots and runs of separators are kept, so files -// named zope.interface-* and zope_interface-* end up under two packages. -func init() { - retention.Default.Register(retention.Spec{Key: "pypi", CanonicalPackage: retention.DefaultCanonical("pypi")}) -} - -// NewPyPIHandlerWithUpstreams creates a PyPI handler with custom API and -// package download upstreams. -func NewPyPIHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, downloadURL string) *PyPIHandler { - h := &PyPIHandler{ +// NewPyPIHandler creates a new PyPI protocol handler. +func NewPyPIHandler(proxy *Proxy, proxyURL string) *PyPIHandler { + return &PyPIHandler{ proxy: proxy, - upstreamURL: configuredUpstreamURL(upstreamURL, pypiUpstream), - downloadURL: configuredUpstreamURL(downloadURL, pypiDownloadUpstream), + upstreamURL: pypiUpstream, proxyURL: strings.TrimSuffix(proxyURL, "/"), } - return h } // Routes returns the HTTP handler for PyPI requests. func (h *PyPIHandler) Routes() http.Handler { mux := http.NewServeMux() - // Simple API + // Simple API (used by pip) mux.HandleFunc("GET /simple/", h.handleSimpleIndex) mux.HandleFunc("GET /simple/{name}/", h.handleSimplePackage) @@ -87,7 +55,7 @@ func (h *PyPIHandler) Routes() http.Handler { // Package downloads (cache these) mux.HandleFunc("GET /packages/{path...}", h.handleDownload) - return rejectQueryChars(mux) + return mux } // handleSimpleIndex serves the simple API index. @@ -104,283 +72,96 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request return } - h.proxy.Logger.Debug("pypi simple request", "package", name) + h.proxy.Logger.Info("pypi simple request", "package", name) upstreamURL := fmt.Sprintf("%s/simple/%s/", h.upstreamURL, name) - accept := selectPyPISimpleRepresentation(r.Header.Get("Accept")) - cacheKey := pypiSimpleCacheKey(name, accept) + cacheKey := name + "/simple" - // Rewritten pages go through the rewrite cache, keyed by the upstream - // page, its content type and the versions filtered out. Without cooldown - // the filtered versions are known up front, so a repeat request can be - // answered without reading the stored page. - cooldown := h.proxy.cooldownApplies("pypi", name) - if !cooldown { - denied := h.addDeniedVersions(name, nil) - if stored, contentType, ok := h.proxy.storedRewriteVariant("pypi", cacheKey, h.proxyURL, cacheKey, func(contentType string) []string { - return simpleRewriteVariant(contentType, denied) - }); ok { - ensureVaryAccept(w.Header()) - serveRewrittenMetadataAs(w, r, stored, contentType) - return - } - } - - upstream := h.proxy.fetchMetadataDocument(r.Context(), "pypi", cacheKey, upstreamURL, accept, nil) - if err := upstream.err; err != nil { + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", cacheKey, upstreamURL, "text/html") + if err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) return } - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return } - contentType := upstream.contentType - // When cooldown applies, fetch JSON metadata to get version timestamps + // When cooldown is enabled, fetch JSON metadata to get version timestamps var filteredVersions map[string]bool - if cooldown { - var err error - filteredVersions, err = h.fetchFilteredVersions(r, name) - if err != nil { - h.proxy.writeCooldownLookupError(w, "pypi", name, err) - return - } - } - filteredVersions = h.addDeniedVersions(name, filteredVersions) - - isJSON := isJSONMediaType(contentType) - digest := variantDigest(upstream.digest, upstream.body, simpleRewriteVariant(contentType, filteredVersions)...) - rewritten, err := h.proxy.cachedRewrite(r.Context(), "pypi", h.proxyURL, cacheKey, upstream.body, digest, func(body []byte) ([]byte, []string, error) { - if isJSON { - out, err := h.rewriteSimpleJSON(body, filteredVersions) - return out, nil, err - } - body = h.filterSimpleHTMLLinks(body, filteredVersions) - return h.rewriteSimpleHTML(body, filteredVersions), nil, nil - }) - if err != nil { - if r.Context().Err() != nil { - return // the client left while waiting on a shared rewrite - } - if len(filteredVersions) != 0 { - h.proxy.Logger.Error("failed to filter pypi simple json", "package", name, "error", err) - http.Error(w, "failed to filter package metadata", http.StatusBadGateway) - return - } - h.proxy.Logger.Warn("failed to rewrite pypi simple json, proxying original", "error", err) - w.Header().Set(headerContentType, contentType) - ensureVaryAccept(w.Header()) - markStale(w, upstream.stale) - w.WriteHeader(http.StatusOK) - _, _ = w.Write(upstream.body) - return + if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() { + filteredVersions = h.fetchFilteredVersions(r, name) } - ensureVaryAccept(w.Header()) - markStale(w, upstream.stale) - serveRewrittenMetadataAs(w, r, rewritten, contentType) + rewritten := h.rewriteSimpleHTML(body, filteredVersions) + + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(rewritten) } -// addDeniedVersions adds the package's denylisted versions to filtered, -// allocating it when there are any and it is nil. -func (h *PyPIHandler) addDeniedVersions(name string, filtered map[string]bool) map[string]bool { - for version := range h.proxy.Denylist.Versions(canonicalPackagePURL("pypi", name)) { - if filtered == nil { - filtered = make(map[string]bool) - } - filtered[version] = true - } - return filtered -} - -// simpleRewriteVariant returns what a simple page rewrite depends on besides -// the upstream page: its content type, which picks the HTML or JSON rewrite, -// and the versions filtered out. -func simpleRewriteVariant(contentType string, filtered map[string]bool) []string { - return append([]string{contentType}, slices.Sorted(maps.Keys(filtered))...) -} - -func selectPyPISimpleRepresentation(accept string) string { - if strings.TrimSpace(accept) == "" { - return pypiLegacyHTML - } - - type score struct { - quality float64 - specificity int - matched bool - } - - scores := map[string]score{ - pypiSimpleJSON: {}, - pypiSimpleHTML: {}, - pypiLegacyHTML: {}, - } - - update := func(representation string, quality float64, specificity int) { - current := scores[representation] - if !current.matched || specificity > current.specificity || - (specificity == current.specificity && quality > current.quality) { - scores[representation] = score{quality: quality, specificity: specificity, matched: true} - } - } - - for part := range strings.SplitSeq(accept, ",") { - mediaType, params, err := mime.ParseMediaType(strings.TrimSpace(part)) - if err != nil { - continue - } - - quality := 1.0 - if value, ok := params["q"]; ok { - quality, err = strconv.ParseFloat(value, 64) - if err != nil || quality < 0 || quality > 1 { - continue - } - } - - switch mediaType { - case pypiSimpleJSON, pypiSimpleLatestJSON: - update(pypiSimpleJSON, quality, pypiExactSpecificity) - case pypiSimpleHTML, pypiSimpleLatestHTML: - update(pypiSimpleHTML, quality, pypiExactSpecificity) - case pypiLegacyHTML: - update(pypiLegacyHTML, quality, pypiExactSpecificity) - case "application/*": - update(pypiSimpleJSON, quality, 1) - update(pypiSimpleHTML, quality, 1) - case "text/*": - update(pypiLegacyHTML, quality, 1) - case "*/*": - update(pypiSimpleJSON, quality, 0) - update(pypiSimpleHTML, quality, 0) - update(pypiLegacyHTML, quality, 0) - } - } - - bestMediaType := "" - bestScore := score{} - for _, mediaType := range []string{pypiSimpleJSON, pypiSimpleHTML, pypiLegacyHTML} { - candidate := scores[mediaType] - if !candidate.matched || candidate.quality == 0 { - continue - } - if bestMediaType == "" || candidate.quality > bestScore.quality || - (candidate.quality == bestScore.quality && candidate.specificity > bestScore.specificity) { - bestMediaType = mediaType - bestScore = candidate - } - } - - if bestMediaType == "" || bestScore.specificity == 0 { - return pypiLegacyHTML - } - return bestMediaType -} - -func pypiSimpleCacheKey(name, mediaType string) string { - switch mediaType { - case pypiSimpleJSON: - return name + "/simple/json" - case pypiSimpleHTML: - return name + "/simple/html" - default: - return name + "/simple" - } -} - -func isJSONMediaType(contentType string) bool { - mediaType, _, err := mime.ParseMediaType(contentType) - if err != nil { - return false - } - return mediaType == "application/json" || strings.HasSuffix(mediaType, "+json") -} - -func ensureVaryAccept(header http.Header) { - ensureVary(header, "Accept") -} - -// fetchFilteredVersions fetches JSON metadata and returns a set of version -// strings that should be filtered out due to cooldown. A package the JSON API -// doesn't know has no publish times to go by and gets an empty set; any other -// failure to read them is an error. -func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[string]bool, error) { - times, err := h.fetchReleaseTimes(r, name) - if err != nil { - return nil, err - } - - packagePURL := canonicalPackagePURL("pypi", name) - filtered := make(map[string]bool) - - for version, publishedAt := range times { - if !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) { - filtered[version] = true - } - } - - if len(filtered) == 0 { - return nil, nil - } - return filtered, nil -} - -// fetchReleaseTimes reads the JSON API and returns each release's newest -// upload time, zero for a release without one. A package the JSON API -// doesn't know gets nil; any other failure is an error. -func (h *PyPIHandler) fetchReleaseTimes(r *http.Request, name string) (map[string]time.Time, error) { +// fetchFilteredVersions fetches JSON metadata and returns a set of version strings +// that should be filtered out due to cooldown. +func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[string]bool { jsonURL := fmt.Sprintf("%s/pypi/%s/json", h.upstreamURL, name) - - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", name+"/json", jsonURL) - if errors.Is(err, ErrUpstreamNotFound) { - return nil, nil - } + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, jsonURL, nil) if err != nil { - return nil, err + return nil + } + req.Header.Set("Accept", "application/json") + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return nil + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil } var metadata map[string]any - if err := json.Unmarshal(body, &metadata); err != nil { - return nil, fmt.Errorf("parsing pypi json metadata: %w", err) + if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { + return nil } releases, ok := metadata["releases"].(map[string]any) if !ok { - return nil, errors.New("pypi json metadata has no releases") + return nil } - times := make(map[string]time.Time, len(releases)) + packagePURL := purl.MakePURLString("pypi", name, "") + filtered := make(map[string]bool) + for version, files := range releases { filesArr, ok := files.([]any) if !ok { continue } - times[version] = h.newestUploadTime(filesArr) + publishedAt := h.newestUploadTime(filesArr) + if !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) { + filtered[version] = true + } } - return times, nil -} -var ( - // simpleLinkRe matches a whole anchor tag: filename. - simpleLinkRe = regexp.MustCompile(`]+href="[^"]*"[^>]*>[^<]+`) - // simpleLinkTextRe captures an anchor's text. - simpleLinkTextRe = regexp.MustCompile(`>([^<]+)`) -) + if len(filtered) == 0 { + return nil + } + return filtered +} // rewriteSimpleHTML rewrites package URLs in simple API HTML to point at this proxy. // If filteredVersions is non-nil, links for those versions are removed entirely. func (h *PyPIHandler) rewriteSimpleHTML(body []byte, filteredVersions map[string]bool) []byte { // If cooldown filtering is active, remove entire tags for filtered versions if len(filteredVersions) > 0 { - body = simpleLinkRe.ReplaceAllFunc(body, func(match []byte) []byte { + // Match full anchor tags: filename + linkRe := regexp.MustCompile(`]+href="[^"]*"[^>]*>[^<]+`) + body = linkRe.ReplaceAllFunc(body, func(match []byte) []byte { // Extract filename from between tags - innerMatch := simpleLinkTextRe.FindSubmatch(match) + innerRe := regexp.MustCompile(`>([^<]+)`) + innerMatch := innerRe.FindSubmatch(match) if len(innerMatch) < minSubmatchParts { return match } @@ -393,96 +174,26 @@ func (h *PyPIHandler) rewriteSimpleHTML(body []byte, filteredVersions map[string }) } - return h.rewriteDownloadHrefs(body) -} + // Match href attributes pointing to packages + // PyPI URLs look like: https://files.pythonhosted.org/packages/... + re := regexp.MustCompile(`href="(https://files\.pythonhosted\.org/packages/[^"]+)"`) -// rewriteDownloadHrefs points href attributes for files on the configured -// download host at the proxy. It makes the same replacements as the regular -// expression `href="(/packages/[^"]+)"` would: an href -// whose URL starts with the download host's /packages/ and has at least one -// more character before the closing quote. Finding the literal prefix is many -// times faster than running the regular expression over every link. -func (h *PyPIHandler) rewriteDownloadHrefs(body []byte) []byte { - from := []byte(`href="` + h.downloadURL + `/packages/`) - // The download route is /pypi/packages/{path...} and keeps the path - // from the download host, /packages/ included. - to := []byte(`href="` + h.proxyURL + `/pypi/packages/packages/`) - - var out []byte - done := 0 // body[:done] is in out already - for search := 0; ; { - i := bytes.Index(body[search:], from) - if i < 0 { - break - } - start := search + i - rest := start + len(from) - end := bytes.IndexByte(body[rest:], '"') - if end <= 0 { - // No closing quote, or nothing before it: not a match. The - // regular expression would go on looking from the next byte. - search = start + 1 - continue - } - if out == nil { - out = make([]byte, 0, len(body)+len(body)/8) - } - out = append(out, body[done:start]...) - out = append(out, to...) - done = rest - search = rest + end + 1 - } - if out == nil { - return body - } - return append(out, body[done:]...) -} - -func (h *PyPIHandler) rewriteSimpleJSON(body []byte, filteredVersions map[string]bool) ([]byte, error) { - var metadata map[string]any - if err := json.Unmarshal(body, &metadata); err != nil { - return nil, err - } - - files, ok := metadata["files"].([]any) - if !ok { - return nil, errors.New("pypi simple json response has no files array") - } - - rewrittenFiles := make([]any, 0, len(files)) - for _, file := range files { - entry, ok := file.(map[string]any) - if !ok { - rewrittenFiles = append(rewrittenFiles, file) - continue + return re.ReplaceAllFunc(body, func(match []byte) []byte { + submatch := re.FindSubmatch(match) + if len(submatch) < minSubmatchParts { + return match } - if filename, ok := entry["filename"].(string); ok { - _, version := h.parseFilename(filename) - if version != "" && filteredVersions[version] { - continue - } + origURL := string(submatch[1]) + + u, err := url.Parse(origURL) + if err != nil { + return match } - h.rewriteURLEntry(entry) - rewrittenFiles = append(rewrittenFiles, entry) - } - - metadata["files"] = rewrittenFiles - - // The versions list (PEP 700) loses the filtered versions too, whose - // files were all left out above. - if versions, ok := metadata["versions"].([]any); ok && len(filteredVersions) > 0 { - keptVersions := make([]any, 0, len(versions)) - for _, v := range versions { - if version, ok := v.(string); ok && filteredVersions[version] { - continue - } - keptVersions = append(keptVersions, v) - } - metadata["versions"] = keptVersions - } - return json.Marshal(metadata) + newURL := fmt.Sprintf("%s/pypi/packages%s", h.proxyURL, u.Path) + return []byte(fmt.Sprintf(`href="%s"`, newURL)) + }) } // handleJSON serves the JSON API package metadata. @@ -493,7 +204,7 @@ func (h *PyPIHandler) handleJSON(w http.ResponseWriter, r *http.Request) { return } - h.proxy.Logger.Debug("pypi json request", "package", name) + h.proxy.Logger.Info("pypi json request", "package", name) upstreamURL := fmt.Sprintf("%s/pypi/%s/json", h.upstreamURL, name) h.proxyAndRewriteJSON(w, r, upstreamURL, name+"/json") @@ -508,90 +219,56 @@ func (h *PyPIHandler) handleVersionJSON(w http.ResponseWriter, r *http.Request) http.Error(w, "invalid request", http.StatusBadRequest) return } - if h.proxy.versionDenied("pypi", name, version) { - http.Error(w, "not found", http.StatusNotFound) - return - } - h.proxy.Logger.Debug("pypi version json request", "package", name, "version", version) + h.proxy.Logger.Info("pypi version json request", "package", name, "version", version) upstreamURL := fmt.Sprintf("%s/pypi/%s/%s/json", h.upstreamURL, name, version) h.proxyAndRewriteJSON(w, r, upstreamURL, name+"/"+version) } // proxyAndRewriteJSON fetches JSON metadata and rewrites download URLs. -// Rewrites go through the rewrite cache, as for npm. func (h *PyPIHandler) proxyAndRewriteJSON(w http.ResponseWriter, r *http.Request, upstreamURL, cacheKey string) { - if stored, ok := h.proxy.storedRewrite("pypi", cacheKey, h.proxyURL, cacheKey); ok { - serveRewrittenMetadata(w, r, stored) - return - } - - upstream := h.proxy.fetchMetadataDocument(r.Context(), "pypi", cacheKey, upstreamURL, contentTypeJSON, nil) - if err := upstream.err; err != nil { + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", cacheKey, upstreamURL) + if err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) return } - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "pypi", h.proxyURL, cacheKey, upstream.body, upstream.digest, h.rewriteJSONMetadata) + rewritten, err := h.rewriteJSONMetadata(body) if err != nil { - if r.Context().Err() != nil { - return // the client left while waiting on a shared rewrite - } - if h.proxy.mustFilterMetadata("pypi", r.PathValue("name")) { - h.proxy.Logger.Error("failed to filter pypi metadata", "url", upstreamURL, "error", err) - http.Error(w, "failed to filter package metadata", http.StatusBadGateway) - return - } h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) - markStale(w, upstream.stale) - w.Header().Set(headerContentType, "application/json") - _, _ = w.Write(upstream.body) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) return } - markStale(w, upstream.stale) - serveRewrittenMetadata(w, r, rewritten) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(rewritten) } // rewriteJSONMetadata rewrites download URLs in PyPI JSON metadata. // If cooldown is enabled, versions published too recently are filtered out. -// It also returns what it kept: the releases, and the current version's files -// unless they were dropped. -func (h *PyPIHandler) rewriteJSONMetadata(body []byte) ([]byte, []string, error) { +func (h *PyPIHandler) rewriteJSONMetadata(body []byte) ([]byte, error) { var metadata map[string]any if err := json.Unmarshal(body, &metadata); err != nil { - return nil, nil, err + return nil, err } packageName, _ := extractPyPIName(metadata) packagePURL := "" if packageName != "" { - packagePURL = canonicalPackagePURL("pypi", packageName) + packagePURL = purl.MakePURLString("pypi", packageName, "") } h.filterAndRewriteReleases(metadata, packageName, packagePURL) h.filterAndRewriteURLs(metadata, packagePURL) - var kept []string - if releases, ok := metadata["releases"].(map[string]any); ok { - kept = slices.Collect(maps.Keys(releases)) - } - if urls, ok := metadata["urls"].([]any); ok && len(urls) > 0 { - // Not a valid release key, so it can't be confused with one. - kept = append(kept, "\x00urls") - } - - out, err := json.Marshal(metadata) - return out, kept, err + return json.Marshal(metadata) } // filterAndRewriteReleases applies cooldown filtering and URL rewriting to the @@ -603,8 +280,8 @@ func (h *PyPIHandler) filterAndRewriteReleases(metadata map[string]any, packageN } for version, files := range releases { - if h.proxy.versionDenied("pypi", packageName, version) || h.shouldFilterRelease(packagePURL, files) { - h.proxy.Logger.Info("policy: filtering pypi version", + if h.shouldFilterRelease(packagePURL, files) { + h.proxy.Logger.Info("cooldown: filtering pypi version", "package", packageName, "version", version) delete(releases, version) continue @@ -626,75 +303,7 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { } publishedAt := h.newestUploadTime(filesArr) - return !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) -} - -// versionInCooldown reports whether a version is still inside the cooldown -// window. Filtering the simple index is not enough on its own: file URLs are -// recorded in lockfiles and requirements pins, so pip can reach the download -// path without ever reading the index. -// -// A release whose upload time cannot be determined is allowed through, matching -// how fetchFilteredVersions treats it. Failing to read the times is an error. -// The time read is stored for storedCooldown. -func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) (bool, error) { - if !h.proxy.cooldownApplies("pypi", name) { - return false, nil - } - - times, err := h.fetchReleaseTimes(r, name) - if err != nil { - return false, err - } - publishedAt := times[version] - if publishedAt.IsZero() { - return false, nil - } - h.storeUploadTime(name, version, publishedAt) - return !h.proxy.cooldown().IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil -} - -// storeUploadTime records a release's newest upload time under the version -// PURL the file is cached under, so storedCooldown finds it. -func (h *PyPIHandler) storeUploadTime(name, version string, publishedAt time.Time) { - if h.proxy.DB == nil { - return - } - packagePURL, versionPURL, err := packagePURLStrings("pypi", name, version) - if err != nil { - return - } - if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, publishedAt); err != nil { - h.proxy.Logger.Warn("cooldown: could not store pypi upload time", - "package", name, "version", version, "error", err) - } -} - -// storedCooldown answers the cooldown check for a download from the upload -// time stored for its release, without reading the JSON API, and reports -// whether it could. A release's time is its newest file's, and files can be -// added to a release later, so the stored time only ever moves forward: -// - a stored time inside the window withholds every file of the release; -// - a stored time outside it only lets through a file that is already -// cached. A file that is not cached may be one uploaded since, so it goes -// to the JSON API, which also stores the release's current time. -func (h *PyPIHandler) storedCooldown(r *http.Request, name, version, filename string) (inCooldown, answered bool) { - if h.proxy.DB == nil || !h.proxy.cooldownApplies("pypi", name) { - return false, false - } - packagePURL, versionPURL, err := packagePURLStrings("pypi", name, version) - if err != nil { - return false, false - } - ver, err := h.proxy.DB.GetVersionByPURL(versionPURL) - if err != nil || ver == nil || !ver.PublishedAt.Valid { - return false, false - } - if !h.proxy.cooldown().IsAllowed("pypi", packagePURL, ver.PublishedAt.Time) { - return true, true - } - cached, err := h.proxy.HasCachedArtifact(r.Context(), "pypi", name, version, filename) - return false, err == nil && cached + return !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) } // rewriteFileEntries rewrites URLs in a list of file entries. @@ -719,10 +328,7 @@ func (h *PyPIHandler) filterAndRewriteURLs(metadata map[string]any, packagePURL return } - info, _ := metadata["info"].(map[string]any) - name, _ := info["name"].(string) - version, _ := info["version"].(string) - if h.proxy.versionDenied("pypi", name, version) || h.shouldFilterRelease(packagePURL, urls) { + if h.shouldFilterRelease(packagePURL, urls) { metadata["urls"] = []any{} } @@ -775,8 +381,15 @@ func (h *PyPIHandler) rewriteURLEntry(entry map[string]any) { return } - if strings.HasPrefix(urlStr, h.downloadURL+"/packages/") { - entry["url"] = h.proxyURL + "/pypi/packages" + strings.TrimPrefix(urlStr, h.downloadURL) + u, err := url.Parse(urlStr) + if err != nil { + return + } + + // Only rewrite pythonhosted.org URLs + if u.Host == "files.pythonhosted.org" { + newURL := fmt.Sprintf("%s/pypi/packages%s", h.proxyURL, u.Path) + entry["url"] = newURL } } @@ -799,211 +412,80 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { filename := parts[len(parts)-1] name, version := h.parseFilename(filename) - if name != "" { - inCooldown, answered := h.storedCooldown(r, name, version, filename) - var err error - if !answered { - inCooldown, err = h.versionInCooldown(r, name, version) - } - if err != nil { - if !h.proxy.serveCachedDespiteCooldownError(w, r, "pypi", name, version, filename, err) { - h.proxy.writeCooldownLookupError(w, "pypi", name, err) - } - return - } - if inCooldown { - h.proxy.Logger.Info("cooldown: withholding pypi file", - "name", name, "version", version, "filename", filename) - http.Error(w, "not found", http.StatusNotFound) - return - } - } - if name == "" { // Can't determine name/version, use hash as identifier name = fmt.Sprintf("_hash_%s", hashPath(path)) version = "0" } - h.proxy.Logger.Debug("pypi download request", + h.proxy.Logger.Info("pypi download request", "name", name, "version", version, "filename", filename) - // The path value starts with 'packages/' (no leading slash), so add - // the separator here. - upstreamURL := fmt.Sprintf("%s/%s", h.downloadURL, path) + // Construct upstream URL; the incoming path starts with + // '/packages' so there is no need to include it in the format + // string + upstreamURL := fmt.Sprintf("https://files.pythonhosted.org/%s", path) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "pypi", name, version, filename, upstreamURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") + h.proxy.Logger.Error("failed to get artifact", "error", err) + http.Error(w, "failed to fetch package", http.StatusBadGateway) return } - ServeArtifactRequest(w, r, result) + ServeArtifact(w, result) } -// archiveExtensions are sdist formats of the form {name}-{version}{ext}. They -// carry no trailing tags, but legacy sdist names may contain hyphens. -var archiveExtensions = []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.Z", ".tgz", ".tar", ".zip"} - -// windowsInstallerExtensions are the legacy distutils bdist_wininst and -// bdist_msi formats, which share a filename layout. -var windowsInstallerExtensions = []string{".exe", ".msi"} - -func (h *PyPIHandler) parseFilename(filename string) (name, version string) { - return PyPIFilenameIdentity(filename) -} - -// PyPIFilenameIdentity extracts the package name and version a PyPI file is -// cached under from its filename, so proxy mirror stores files where the -// download handler looks for them. Handles wheels, sdists and legacy bdist -// formats: +// parseFilename extracts package name and version from a PyPI filename. +// Handles both wheels and sdists: // - requests-2.31.0-py3-none-any.whl // - requests-2.31.0.tar.gz -// - numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg -// - numpy-1.8.0.win32-py2.7.exe -func PyPIFilenameIdentity(filename string) (name, version string) { - // PEP 658/714 core-metadata sidecars are the distribution filename plus - // ".metadata"; they describe the same name and version. Without this, pip's - // metadata-only fetches fall back to a hash-derived package identifier. - filename = strings.TrimSuffix(filename, PyPIMetadataSuffix) - - switch { - case strings.HasSuffix(filename, ".whl"): - return parseWheelFilename(strings.TrimSuffix(filename, ".whl")) - case strings.HasSuffix(filename, ".egg"): - return parseEggFilename(strings.TrimSuffix(filename, ".egg")) - } - - for _, ext := range windowsInstallerExtensions { - if strings.HasSuffix(filename, ext) { - return parseWindowsInstallerFilename(strings.TrimSuffix(filename, ext)) +func (h *PyPIHandler) parseFilename(filename string) (name, version string) { + // Try wheel format first: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl + if strings.HasSuffix(filename, ".whl") { + base := strings.TrimSuffix(filename, ".whl") + parts := strings.Split(base, "-") + if len(parts) >= minWheelParts { + // Find where version ends (version followed by python tag) + for i := 1; i < len(parts)-2; i++ { + // Check if this looks like a python tag (py2, py3, cp39, etc) + if isPythonTag(parts[i]) { + name = strings.Join(parts[:i-1], "-") + version = parts[i-1] + return + } + } } } - for _, ext := range archiveExtensions { + // Try sdist formats: {name}-{version}.tar.gz, {name}-{version}.zip + for _, ext := range []string{".tar.gz", ".tar.bz2", ".zip", ".tar"} { if strings.HasSuffix(filename, ext) { - return splitNameVersion(strings.TrimSuffix(filename, ext)) + base := strings.TrimSuffix(filename, ext) + // Find last hyphen followed by version + for i := len(base) - 1; i >= 0; i-- { + if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) { + return base[:i], base[i+1:] + } + } } } return "", "" } -// parseWheelFilename parses the PEP 427 layout -// {name}-{version}(-{build})?-{python}-{abi}-{platform}, base being the -// filename without its ".whl" suffix. The spec escapes every hyphen in the name -// and version to '_', so the first two fields are authoritative even when the -// optional build tag is present. -func parseWheelFilename(base string) (name, version string) { - parts := strings.Split(base, "-") - if len(parts) < minWheelParts { - return "", "" - } - - return parts[0], parts[1] -} - -// parseEggFilename parses the setuptools bdist_egg layout -// {name}-{version}-py{X.Y}(-{platform})?, base being the filename without its -// ".egg" suffix. setuptools escapes hyphens in the name and version to '_', but -// eggs built by other tooling do not always, so the version is located relative -// to the interpreter field rather than assumed to be the second field. -func parseEggFilename(base string) (name, version string) { - parts := strings.Split(base, "-") - // Scan from the end: the trailing platform fields never look like an - // interpreter tag, so the last match is the real one even when the package - // name itself carries a "py{N}" component. Stop before index 1, since a tag - // any earlier would leave no room for both a name and a version. - for i := len(parts) - 1; i >= minEggParts-1; i-- { - if !isEggPythonTag(parts[i]) || !isVersionField(parts[i-1]) { - continue - } - - return strings.Join(parts[:i-1], "-"), parts[i-1] - } - - // No interpreter field: {name}-{version}. - return splitNameVersion(base) -} - -// parseWindowsInstallerFilename parses the distutils bdist_wininst and -// bdist_msi layout {name}-{version}.{platform}(-py{X.Y})?, base being the -// filename without its ".exe" or ".msi" suffix. The platform is joined to the -// version with a '.' rather than a '-' and may itself contain a hyphen -// ("win-amd64"), so both trailing fields are stripped before the name and -// version are split apart. -func parseWindowsInstallerFilename(base string) (name, version string) { - if i := strings.LastIndex(base, "-py"); i >= 0 && isDottedNumber(base[i+len("-py"):]) { - base = base[:i] - } - - // The platform is the final '.'-separated field. Requiring it to start with - // a non-digit keeps a dotted version from being truncated when a filename - // carries no platform tag. - i := strings.LastIndex(base, ".") - if i < 0 || i+1 >= len(base) || isVersionStart(base[i+1]) { - return "", "" - } - - return splitFullname(base[:i]) -} - -// splitFullname splits the distutils fullname {name}-{version} that precedes a -// Windows installer's platform field. Unlike an sdist, a wininst fullname may -// carry a trailing build variant ("cx_Oracle-5.1.2-11g"), which belongs to -// neither the name nor the version, so the first purely numeric field wins and -// anything after it is discarded. -func splitFullname(fullname string) (name, version string) { - parts := strings.Split(fullname, "-") - for i := 1; i < len(parts); i++ { - if isDottedNumber(parts[i]) { - return strings.Join(parts[:i], "-"), parts[i] - } - } - - // No purely numeric field, e.g. a prerelease version like "1.0b1". - return splitNameVersion(fullname) -} - -// splitNameVersion splits a {name}-{version} pair at the last hyphen that -// starts a version, leaving hyphens inside the name intact. -func splitNameVersion(base string) (name, version string) { - for i := len(base) - 1; i >= 0; i-- { - if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) { - return base[:i], base[i+1:] - } - } - - return "", "" -} - -// isEggPythonTag reports whether field is the py{X.Y} interpreter field that -// setuptools places directly after the version in an egg filename. -func isEggPythonTag(field string) bool { - const prefix = "py" - - return len(field) > len(prefix) && strings.HasPrefix(field, prefix) && isVersionStart(field[len(prefix)]) -} - -// isVersionField reports whether field can be a version, i.e. it is non-empty -// and starts with a digit as every PEP 440 release segment does. -func isVersionField(field string) bool { - return field != "" && isVersionStart(field[0]) -} - -// isDottedNumber reports whether s is a dotted numeric version such as "2.7". -func isDottedNumber(s string) bool { - if s == "" || !isVersionStart(s[0]) { +func isPythonTag(s string) bool { + if len(s) < minPythonTagLen { return false } - - for i := range len(s) { - if !isVersionStart(s[i]) && s[i] != '.' { - return false + // Python tags start with py, cp, pp, ip, jy + prefixes := []string{"py", "cp", "pp", "ip", "jy"} + for _, p := range prefixes { + if strings.HasPrefix(s, p) { + return true } } - - return true + return false } func isVersionStart(c byte) bool { @@ -1024,132 +506,22 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s http.Error(w, "failed to create request", http.StatusInternalServerError) return } - req.Header.Set("Accept", selectPyPISimpleRepresentation(r.Header.Get("Accept"))) + req.Header.Set("Accept", "text/html") resp, err := h.proxy.HTTPClient.Do(req) if err != nil { - if writeUpstreamHeld(w, err) { - return - } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return } defer func() { _ = resp.Body.Close() }() - // The HTML root index links projects by absolute path, such as - // /simple//, which must point at the proxy's /pypi mount instead. - // It is large, so it is rewritten as it streams. - if resp.StatusCode == http.StatusOK && resp.Header.Get("Content-Encoding") == "" && - !strings.Contains(resp.Header.Get(headerContentType), "json") { - prefix := "/simple/" - if u, err := url.Parse(h.upstreamURL); err == nil { - prefix = strings.TrimSuffix(u.Path, "/") + prefix - } - to := h.proxyURL + "/pypi/simple/" - resp.Body = &replacingReader{ - src: resp.Body, - replacements: []replacement{ - {from: []byte(`href="` + prefix), to: []byte(`href="` + to)}, - {from: []byte(`href='` + prefix), to: []byte(`href='` + to)}, - }, - } - resp.Header.Del(headerContentLength) - resp.Header.Del("ETag") // it names the upstream body - resp.ContentLength = -1 - } - - h.proxy.relayResponse(w, r, resp, func(dst, src http.Header) { - copyRelayHeaders(dst, src) - ensureVaryAccept(dst) - }) -} - -const replacingReaderBufferSize = 32 << 10 - -// replacingReader makes each replacement in src as it is read. -type replacingReader struct { - src io.ReadCloser - replacements []replacement - in []byte // read from src, not yet scanned - out []byte // scanned, not yet returned - buf []byte - err error -} - -type replacement struct{ from, to []byte } - -func (r *replacingReader) Read(p []byte) (int, error) { - for len(r.out) == 0 { - if r.err != nil { - return 0, r.err - } - if r.buf == nil { - r.buf = make([]byte, replacingReaderBufferSize) - } - n, err := r.src.Read(r.buf) - r.in = append(r.in, r.buf[:n]...) - r.err = err - // Keep a tail that could be the start of a match split across reads. - keep := 0 - if err == nil { - for _, rep := range r.replacements { - keep = max(keep, len(rep.from)-1) - } - } - scan := max(len(r.in)-keep, 0) - out, done := r.replace(scan) - if done < scan { - out = append(out, r.in[done:scan]...) - done = scan - } - r.out = out - r.in = append([]byte(nil), r.in[done:]...) - } - n := copy(p, r.out) - r.out = r.out[n:] - return n, nil -} - -// replace makes the replacements that start before scan in r.in, returning -// the output and how much of r.in it covers. -func (r *replacingReader) replace(scan int) (out []byte, done int) { - // at holds each replacement's next match at or after done, or -1. - // Only a match the last replacement consumed is searched for again, - // so each pass over r.in stays linear. - at := make([]int, len(r.replacements)) - for i := range at { - at[i] = indexFrom(r.in, r.replacements[i].from, 0) - } - for { - k := -1 - for i, j := range at { - if j >= 0 && (k < 0 || j < at[k]) { - k = i - } - } - if k < 0 || at[k] >= scan { - break - } - rep := &r.replacements[k] - out = append(out, r.in[done:at[k]]...) - out = append(out, rep.to...) - done = at[k] + len(rep.from) - for i, j := range at { - if j >= 0 && j < done { - at[i] = indexFrom(r.in, r.replacements[i].from, done) - } + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) } } - return out, done -} -func (r *replacingReader) Close() error { return r.src.Close() } - -// indexFrom returns the index of the first sep in b at or after from, or -1. -func indexFrom(b, sep []byte, from int) int { - if i := bytes.Index(b[from:], sep); i >= 0 { - return from + i - } - return -1 + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) } diff --git a/internal/handler/pypi_denylist.go b/internal/handler/pypi_denylist.go deleted file mode 100644 index f39c0d8..0000000 --- a/internal/handler/pypi_denylist.go +++ /dev/null @@ -1,59 +0,0 @@ -package handler - -import ( - "bytes" - "net/url" - "path" - - "golang.org/x/net/html" -) - -// filterSimpleHTMLLinks reads hrefs rather than display text. Tokenization -// handles single quotes, entity escaping, and nested link text without -// reserializing the rest of the index. -func (h *PyPIHandler) filterSimpleHTMLLinks(body []byte, versions map[string]bool) []byte { - if len(versions) == 0 { - return body - } - var result bytes.Buffer - z := html.NewTokenizer(bytes.NewReader(body)) - skip := false - for { - tokenType := z.Next() - if tokenType == html.ErrorToken { - return result.Bytes() - } - raw := append([]byte(nil), z.Raw()...) - if tokenType == html.StartTagToken || tokenType == html.SelfClosingTagToken { - token := z.Token() - if token.Data == "a" { - skip = h.deniedSimpleLink(token, versions) - } - } else if tokenType == html.EndTagToken && z.Token().Data == "a" { - if skip { - skip = false - continue - } - } - if !skip { - result.Write(raw) - } - } -} - -func (h *PyPIHandler) deniedSimpleLink(token html.Token, versions map[string]bool) bool { - for _, attr := range token.Attr { - if attr.Key != "href" { - continue - } - u, err := url.Parse(attr.Val) - if err != nil { - continue - } - _, version := h.parseFilename(path.Base(u.Path)) - if versions[version] { - return true - } - } - return false -} diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index a80e95b..2b58960 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -2,310 +2,18 @@ package handler import ( "encoding/json" - "fmt" "io" "log/slog" - "math/rand/v2" "net/http" "net/http/httptest" - "strconv" "strings" - "sync/atomic" "testing" - "testing/iotest" "time" "github.com/git-pkgs/cooldown" "github.com/git-pkgs/registries/fetch" ) -const uvPyPIAccept = "application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html;q=0.2, text/html;q=0.01" - -type pypiRoundTripFunc func(*http.Request) (*http.Response, error) - -func (f pypiRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { - return f(r) -} - -func pypiHTTPResponse(r *http.Request, contentType, body string) *http.Response { - return &http.Response{ - StatusCode: http.StatusOK, - Status: "200 OK", - Header: http.Header{"Content-Type": []string{contentType}}, - Body: io.NopCloser(strings.NewReader(body)), - ContentLength: int64(len(body)), - Request: r, - } -} - -func setupPyPIHandler(t testing.TB, transport pypiRoundTripFunc) (*PyPIHandler, *Proxy) { - t.Helper() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = &http.Client{Transport: transport} - h := NewPyPIHandlerWithUpstreams(proxy, "http://proxy.test", "", "") - h.upstreamURL = "https://pypi.test" - return h, proxy -} - -func TestSelectPyPISimpleRepresentation(t *testing.T) { - tests := []struct { - name string - accept string - want string - }{ - {"missing header uses legacy html", "", "text/html"}, - {"wildcard uses legacy html", "*/*", "text/html"}, - {"uv prefers json", uvPyPIAccept, pypiSimpleJSON}, - {"json only", pypiSimpleJSON, pypiSimpleJSON}, - {"latest json", pypiSimpleLatestJSON, pypiSimpleJSON}, - {"vendor html", pypiSimpleHTML, pypiSimpleHTML}, - {"higher html quality", pypiSimpleJSON + ";q=0.2, text/html;q=0.8", "text/html"}, - {"json excluded", pypiSimpleJSON + ";q=0, text/html", "text/html"}, - {"application wildcard", "application/*", pypiSimpleJSON}, - {"unsupported type uses legacy html", "application/xml", "text/html"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := selectPyPISimpleRepresentation(tt.accept); got != tt.want { - t.Errorf("selectPyPISimpleRepresentation(%q) = %q, want %q", tt.accept, got, tt.want) - } - }) - } -} - -func TestPyPISimplePackageNegotiatesJSON(t *testing.T) { - const upstreamBody = `{ - "meta":{"api-version":"1.4"}, - "name":"ruff", - "files":[{ - "filename":"ruff-0.16.0-py3-none-any.whl", - "url":"https://files.pythonhosted.org/packages/ab/cd/ruff-0.16.0-py3-none-any.whl", - "hashes":{"sha256":"abc123"}, - "upload-time":"2026-08-01T12:00:00Z" - }] - }` - - var upstreamAccept string - h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - upstreamAccept = r.Header.Get("Accept") - if r.URL.Path != "/simple/ruff/" { - t.Fatalf("upstream path = %q, want %q", r.URL.Path, "/simple/ruff/") - } - return pypiHTTPResponse(r, pypiSimpleJSON, upstreamBody), nil - }) - - req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) - req.Header.Set("Accept", uvPyPIAccept) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - if upstreamAccept != pypiSimpleJSON { - t.Errorf("upstream Accept = %q, want %q", upstreamAccept, pypiSimpleJSON) - } - if got := w.Header().Get("Content-Type"); got != pypiSimpleJSON { - t.Errorf("Content-Type = %q, want %q", got, pypiSimpleJSON) - } - if got := w.Header().Get("Vary"); !strings.Contains(got, "Accept") { - t.Errorf("Vary = %q, want Accept", got) - } - - var result struct { - Meta map[string]string `json:"meta"` - Files []struct { - URL string `json:"url"` - UploadTime string `json:"upload-time"` - } `json:"files"` - } - if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { - t.Fatalf("decode response: %v", err) - } - if result.Meta["api-version"] != "1.4" { - t.Errorf("api-version = %q, want 1.4", result.Meta["api-version"]) - } - if len(result.Files) != 1 { - t.Fatalf("files = %d, want 1", len(result.Files)) - } - if got, want := result.Files[0].URL, "http://proxy.test/pypi/packages/packages/ab/cd/ruff-0.16.0-py3-none-any.whl"; got != want { - t.Errorf("file URL = %q, want %q", got, want) - } - if got := result.Files[0].UploadTime; got != "2026-08-01T12:00:00Z" { - t.Errorf("upload-time = %q, want %q", got, "2026-08-01T12:00:00Z") - } -} - -func TestPyPISimpleIndexNegotiatesJSON(t *testing.T) { - const upstreamBody = `{"meta":{"api-version":"1.4"},"projects":[{"name":"ruff"}]}` - - var upstreamAccept string - h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - upstreamAccept = r.Header.Get("Accept") - return pypiHTTPResponse(r, pypiSimpleJSON, upstreamBody), nil - }) - - req := httptest.NewRequest(http.MethodGet, "/simple/", nil) - req.Header.Set("Accept", uvPyPIAccept) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - if upstreamAccept != pypiSimpleJSON { - t.Errorf("upstream Accept = %q, want %q", upstreamAccept, pypiSimpleJSON) - } - if got := w.Header().Get("Content-Type"); got != pypiSimpleJSON { - t.Errorf("Content-Type = %q, want %q", got, pypiSimpleJSON) - } - if got := w.Header().Get("Vary"); !strings.Contains(got, "Accept") { - t.Errorf("Vary = %q, want Accept", got) - } - if got := w.Body.String(); got != upstreamBody { - t.Errorf("body = %q, want %q", got, upstreamBody) - } -} - -func TestPyPISimplePackageKeepsHTMLDefault(t *testing.T) { - const upstreamBody = `ruff-0.16.0.tar.gz` - - var upstreamAccept string - h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - upstreamAccept = r.Header.Get("Accept") - return pypiHTTPResponse(r, "text/html", upstreamBody), nil - }) - - req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - if upstreamAccept != "text/html" { - t.Errorf("upstream Accept = %q, want text/html", upstreamAccept) - } - if got := w.Header().Get("Content-Type"); got != "text/html" { - t.Errorf("Content-Type = %q, want text/html", got) - } - if !strings.Contains(w.Body.String(), `href="http://proxy.test/pypi/packages/packages/ab/cd/ruff-0.16.0.tar.gz"`) { - t.Errorf("download URL was not rewritten: %s", w.Body.String()) - } -} - -func TestPyPISimplePackageCachesRepresentationsSeparately(t *testing.T) { - hits := make(map[string]int) - h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - accept := r.Header.Get("Accept") - hits[accept]++ - if accept == pypiSimpleJSON { - body := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[]}` - return pypiHTTPResponse(r, pypiSimpleJSON, body), nil - } - return pypiHTTPResponse(r, accept, `ruff.tar.gz`), nil - }) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - - for range 2 { - for _, accept := range []string{pypiLegacyHTML, pypiSimpleHTML, uvPyPIAccept} { - req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) - req.Header.Set("Accept", accept) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("Accept %q: status = %d, want 200: %s", accept, w.Code, w.Body.String()) - } - } - } - - if got := hits[pypiLegacyHTML]; got != 1 { - t.Errorf("HTML upstream requests = %d, want 1", got) - } - if got := hits[pypiSimpleHTML]; got != 1 { - t.Errorf("vendor HTML upstream requests = %d, want 1", got) - } - if got := hits[pypiSimpleJSON]; got != 1 { - t.Errorf("JSON upstream requests = %d, want 1", got) - } -} - -func TestPyPISimpleJSONCooldown(t *testing.T) { - now := time.Now() - old := now.Add(-30 * 24 * time.Hour).Format(time.RFC3339) - recent := now.Add(-time.Hour).Format(time.RFC3339) - - h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - switch r.URL.Path { - case "/simple/ruff/": - body := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[` + - `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz","upload-time":"` + old + `"},` + - `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload-time":"` + recent + `"}` + - `]}` - return pypiHTTPResponse(r, pypiSimpleJSON, body), nil - case "/pypi/ruff/json": - body := `{"releases":{` + - `"1.0.0":[{"upload_time_iso_8601":"` + old + `"}],` + - `"2.0.0":[{"upload_time_iso_8601":"` + recent + `"}]` + - `}}` - return pypiHTTPResponse(r, "application/json", body), nil - default: - t.Fatalf("unexpected upstream path: %s", r.URL.Path) - return nil, nil - } - }) - proxy.Cooldown = &cooldown.Config{Default: "7d"} - - req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) - req.Header.Set("Accept", uvPyPIAccept) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) - } - var result struct { - Files []struct { - Filename string `json:"filename"` - } `json:"files"` - } - if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { - t.Fatalf("decode response: %v", err) - } - if len(result.Files) != 1 || result.Files[0].Filename != "ruff-1.0.0.tar.gz" { - t.Errorf("files = %#v, want only ruff-1.0.0.tar.gz", result.Files) - } -} - -// TestPyPISimpleJSONFiltersVersions checks that the PEP 700 versions list -// loses a version whose files were all filtered out, and keeps one that -// still has files. -func TestPyPISimpleJSONFiltersVersions(t *testing.T) { - h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}, proxyURL: "http://proxy.test"} - body := `{"meta":{"api-version":"1.1"},"name":"ruff","versions":["1.0.0","2.0.0","3.0.0"],"files":[` + - `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz"},` + - `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz"},` + - `{"filename":"ruff-2.0.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/ef/ruff-2.0.0-py3-none-any.whl"}` + - `]}` - out, err := h.rewriteSimpleJSON([]byte(body), map[string]bool{"2.0.0": true, "3.0.0": true}) - if err != nil { - t.Fatal(err) - } - var result struct { - Versions []string `json:"versions"` - } - if err := json.Unmarshal(out, &result); err != nil { - t.Fatal(err) - } - // 3.0.0 has no files to filter, but it is still held back. - if strings.Join(result.Versions, ",") != "1.0.0" { - t.Errorf("versions = %v, want [1.0.0]", result.Versions) - } -} - func TestPyPIParseFilename(t *testing.T) { h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}} @@ -320,58 +28,13 @@ func TestPyPIParseFilename(t *testing.T) { {"aws-sdk-1.0.0.tar.gz", "aws-sdk", "1.0.0"}, {"zipp-3.17.0.zip", "zipp", "3.17.0"}, - // Additional sdist archive formats - {"lxml-4.9.3.tar.xz", "lxml", "4.9.3"}, - {"docutils-0.20.1.tgz", "docutils", "0.20.1"}, - {"psycopg2-2.9.9.tar.bz2", "psycopg2", "2.9.9"}, - // Wheel formats {"requests-2.31.0-py3-none-any.whl", "requests", "2.31.0"}, {"numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.whl", "numpy", "1.26.2"}, {"cryptography-41.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", "cryptography", "41.0.5"}, - // Wheels with a build tag must not fold the tag into the version - {"foo-1.0-1-py3-none-any.whl", "foo", "1.0"}, - {"tensorflow-2.15.0-2-cp311-cp311-manylinux_2_17_x86_64.whl", "tensorflow", "2.15.0"}, - - // PEP 658 core-metadata sidecars resolve to the distribution they describe - {"backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata", "backports_asyncio_runner", "1.2.0"}, - {"requests-2.31.0-py3-none-any.whl.metadata", "requests", "2.31.0"}, - {"requests-2.31.0.tar.gz.metadata", "requests", "2.31.0"}, - - // Eggs: {name}-{version}-py{X.Y}(-{platform})?.egg. Unescaped hyphens in - // the name must not be mistaken for the field separator before the version. - {"numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg", "numpy", "1.8.0"}, - {"aws-sdk-1.0.0-py3.11.egg", "aws-sdk", "1.0.0"}, - {"aws-sdk-1.0.0-py2.7-macosx-10.9-x86_64.egg", "aws-sdk", "1.0.0"}, - {"aws-sdk-1.0.0.egg", "aws-sdk", "1.0.0"}, - // A "py{N}" component inside the name is not the interpreter field, so - // the interpreter must be located from the end of the filename. - {"django-rest-py3-1.0-py3.6.egg", "django-rest-py3", "1.0"}, - - // Windows installers: {name}-{version}.{platform}(-py{X.Y})?.{exe,msi}. - // The platform is not part of the version, and may contain a hyphen. - {"foo-1.0.win32-py2.0.exe", "foo", "1.0"}, - {"pywin32-223.win32-py2.7.exe", "pywin32", "223"}, - {"numpy-1.8.0.win-amd64-py2.7.exe", "numpy", "1.8.0"}, - {"aws-sdk-1.0.0.win32-py2.7.exe", "aws-sdk", "1.0.0"}, - {"pywin32-223.win32.exe", "pywin32", "223"}, - {"cx_Oracle-5.1.2.win32-py2.7.msi", "cx_Oracle", "5.1.2"}, - {"numpy-1.8.0.win-amd64.msi", "numpy", "1.8.0"}, - // A trailing build variant belongs to neither the name nor the version. - {"cx_Oracle-5.1.2-11g.win32-py2.7.exe", "cx_Oracle", "5.1.2"}, - // A prerelease version has no purely numeric field to anchor on. - {"foo-1.0b1.win32-py2.7.exe", "foo", "1.0b1"}, - // Invalid {"invalid", "", ""}, - {"invalid.metadata", "", ""}, - {"backports.ssl_match_hostname-3.4.0.2-py2.7.whl", "", ""}, - {"invalid.exe", "", ""}, - {"foo-1.0.exe", "", ""}, - // An egg with an interpreter field but no version must not promote the - // trailing component of a hyphenated name to the version. - {"aws-sdk-py2.7.egg", "", ""}, } for _, tt := range tests { @@ -405,7 +68,7 @@ func TestPyPIRewriteJSONMetadataCooldown(t *testing.T) { "urls": [{"url": "https://files.pythonhosted.org/packages/ab/cd/requests-2.31.0.tar.gz", "upload_time_iso_8601": "` + recent + `"}] }` - output, _, err := h.rewriteJSONMetadata([]byte(input)) + output, err := h.rewriteJSONMetadata([]byte(input)) if err != nil { t.Fatalf("rewriteJSONMetadata failed: %v", err) } @@ -431,24 +94,25 @@ func TestPyPIRewriteJSONMetadataCooldown(t *testing.T) { } } -// TestPyPIParseFilenameNoHashFallback guards the identifier used for caching: -// a filename that parses to an empty name makes handleDownload fall back to a -// "_hash_" package name, which surfaces as a bogus PURL in the package -// overview. -func TestPyPIParseFilenameNoHashFallback(t *testing.T) { - h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}} - - filenames := []string{ - "backports_asyncio_runner-1.2.0-py3-none-any.whl", - "backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata", - "backports_asyncio_runner-1.2.0.tar.gz", +func TestIsPythonTag(t *testing.T) { + tests := []struct { + tag string + want bool + }{ + {"py3", true}, + {"py2", true}, + {"cp311", true}, + {"cp37", true}, + {"pp39", true}, + {"none", false}, + {"any", false}, + {"manylinux", false}, } - for _, filename := range filenames { - name, version := h.parseFilename(filename) - if name != "backports_asyncio_runner" || version != "1.2.0" { - t.Errorf("parseFilename(%q) = (%q, %q), want (%q, %q)", - filename, name, version, "backports_asyncio_runner", "1.2.0") + for _, tt := range tests { + got := isPythonTag(tt.tag) + if got != tt.want { + t.Errorf("isPythonTag(%q) = %v, want %v", tt.tag, got, tt.want) } } } @@ -460,7 +124,7 @@ func TestPyPIHandler_DownloadUpstreamURL(t *testing.T) { ContentType: "application/octet-stream", } - h := NewPyPIHandlerWithUpstreams(proxy, "http://localhost", "", "") + h := NewPyPIHandler(proxy, "http://localhost") srv := httptest.NewServer(h.Routes()) defer srv.Close() @@ -488,7 +152,7 @@ func TestPyPIHandler_DownloadCacheHit(t *testing.T) { seedPackage(t, db, store, "pypi", "requests", "2.31.0", "requests-2.31.0-py3-none-any.whl", "wheel binary data") - h := NewPyPIHandlerWithUpstreams(proxy, "http://localhost", "", "") + h := NewPyPIHandler(proxy, "http://localhost") srv := httptest.NewServer(h.Routes()) defer srv.Close() @@ -514,7 +178,7 @@ func TestPyPIHandler_DownloadCacheMiss(t *testing.T) { ContentType: "application/octet-stream", } - h := NewPyPIHandlerWithUpstreams(proxy, "http://localhost", "", "") + h := NewPyPIHandler(proxy, "http://localhost") srv := httptest.NewServer(h.Routes()) defer srv.Close() @@ -528,242 +192,3 @@ func TestPyPIHandler_DownloadCacheMiss(t *testing.T) { t.Error("expected fetcher to be called on cache miss") } } - -func TestPyPIDownloadCooldown(t *testing.T) { - now := time.Now() - releases := `{"releases": { - "1.0.0": [{"upload_time_iso_8601": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `"}], - "2.0.0": [{"upload_time_iso_8601": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `"}] - }}` - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", contentTypeJSON) - _, _ = io.WriteString(w, releases) - })) - defer upstream.Close() - - tests := []struct { - name string - filename string - wantStatus int - }{ - {"published before the window serves the file", "newpkg-1.0.0.tar.gz", http.StatusOK}, - {"published inside the window is withheld", "newpkg-2.0.0.tar.gz", http.StatusNotFound}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "7d"} - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("sdist data")), - ContentType: "application/octet-stream", - } - - h := &PyPIHandler{ - proxy: proxy, - upstreamURL: upstream.URL, - proxyURL: "http://localhost", - } - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/packages/packages/ab/cd/ef0123456789/" + tt.filename) - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != tt.wantStatus { - t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus) - } - if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled { - t.Error("fetched a version that is still inside the cooldown window") - } - }) - } -} - -// TestPyPIDownloadCooldownMetadataCache ensures that repeated downloads that -// trigger cooldown filtering reuse the cached PyPI JSON metadata instead of -// fetching it from upstream once per download. -func TestPyPIDownloadCooldownMetadataCache(t *testing.T) { - now := time.Now() - releases := `{"releases": { - "1.0.0": [{"upload_time_iso_8601": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `"}], - "2.0.0": [{"upload_time_iso_8601": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `"}] - }}` - - var metadataRequests atomic.Int64 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/pypi/newpkg/json" { - metadataRequests.Add(1) - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, releases) - return - } - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = io.WriteString(w, "package data") - })) - defer upstream.Close() - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - proxy.Cooldown = &cooldown.Config{Default: "7d"} - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("package data")), - ContentType: "application/octet-stream", - } - - h := &PyPIHandler{ - proxy: proxy, - upstreamURL: upstream.URL, - proxyURL: "http://localhost", - } - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - // Two downloads of the same package: one outside the cooldown window - // (served) and one inside (withheld). Both go through the download path - // that resolves filtered versions. - for _, filename := range []string{"newpkg-1.0.0.tar.gz", "newpkg-2.0.0.tar.gz"} { - resp, err := http.Get(srv.URL + "/packages/packages/ab/cd/ef0123456789/" + filename) - if err != nil { - t.Fatalf("request failed: %v", err) - } - _ = resp.Body.Close() - } - - if got := metadataRequests.Load(); got != 1 { - t.Errorf("upstream metadata JSON requests = %d, want 1 (repeated downloads should reuse the cached metadata)", got) - } -} - -func TestPyPISimpleIndexRewritesProjectLinks(t *testing.T) { - for _, tc := range []struct{ name, upstream, prefix string }{ - {"root", "https://pypi.test", "/simple/"}, - {"subpath", "https://nexus.test/repository/pypi", "/repository/pypi/simple/"}, - } { - t.Run(tc.name, func(t *testing.T) { - // Long enough that links cross the rewriter's read boundaries. - var upstream, want strings.Builder - upstream.WriteString(``) - want.WriteString(``) - for i := range 3000 { - fmt.Fprintf(&upstream, `pkg-%d`+"\n", tc.prefix, i, i) - fmt.Fprintf(&want, `pkg-%d`+"\n", i, i) - } - fmt.Fprintf(&upstream, `q`, tc.prefix) - want.WriteString(`q`) - upstream.WriteString(``) - want.WriteString(``) - - h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - resp := pypiHTTPResponse(r, "text/html", upstream.String()) - resp.Header.Set("Content-Length", strconv.Itoa(upstream.Len())) - resp.Header.Set("ETag", `"upstream"`) - return resp, nil - }) - h.upstreamURL = tc.upstream - - req := httptest.NewRequest(http.MethodGet, "/simple/", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200", w.Code) - } - if got := w.Body.String(); got != want.String() { - t.Errorf("body not rewritten; starts %q", got[:min(len(got), 120)]) - } - if got := w.Header().Get("Content-Length"); got != "" && got != strconv.Itoa(want.Len()) { - t.Errorf("Content-Length = %q, want %d or unset", got, want.Len()) - } - // The upstream's ETag names the body before the rewrite. - if got := w.Header().Get("ETag"); got != "" { - t.Errorf("ETag = %q, want none on a rewritten body", got) - } - }) - } -} - -func TestReplacingReaderSplitReads(t *testing.T) { - const in = `abxhref="/simplehref="/simple/href='/simple/y/'` - const want = `abxhref="/simplehref="P/href='Q/y/'` - r := &replacingReader{ - src: io.NopCloser(iotest.OneByteReader(strings.NewReader(in))), - replacements: []replacement{ - {from: []byte(`href="/simple/`), to: []byte(`href="P/`)}, - {from: []byte(`href='/simple/`), to: []byte(`href='Q/`)}, - }, - } - got, err := io.ReadAll(r) - if err != nil || string(got) != want { - t.Errorf("got %q, %v; want %q", got, err, want) - } -} - -// replaceReference makes the replacements the way replacingReader should: -// left to right, the earliest match first, and the first listed on a tie. -func replaceReference(in string, reps []replacement) string { - var out strings.Builder - for len(in) > 0 { - at, found := -1, -1 - for i, rep := range reps { - if j := strings.Index(in, string(rep.from)); j >= 0 && (at < 0 || j < at) { - at, found = j, i - } - } - if at < 0 { - out.WriteString(in) - break - } - out.WriteString(in[:at]) - out.Write(reps[found].to) - in = in[at+len(reps[found].from):] - } - return out.String() -} - -func TestReplacingReaderMatchesReference(t *testing.T) { - reps := []replacement{ - {from: []byte(`href="/s/`), to: []byte(`href="P/`)}, - {from: []byte(`href='/s/`), to: []byte(`href='Q/`)}, - } - pieces := []string{`href="/s/`, `href='/s/`, `href="/s`, `href=`, `x`, `/s/`, `'`, `"`} - rng := rand.New(rand.NewPCG(1, 2)) - for range 500 { - var b strings.Builder - for range rng.IntN(40) { - b.WriteString(pieces[rng.IntN(len(pieces))]) - } - in := b.String() - for _, src := range []io.Reader{strings.NewReader(in), iotest.OneByteReader(strings.NewReader(in))} { - got, err := io.ReadAll(&replacingReader{src: io.NopCloser(src), replacements: reps}) - if want := replaceReference(in, reps); err != nil || string(got) != want { - t.Fatalf("in %q: got %q, %v; want %q", in, got, err, want) - } - } - } -} - -// BenchmarkReplacingReaderRootIndex reads a root index of double-quoted -// links, where the single-quote pattern never matches. -func BenchmarkReplacingReaderRootIndex(b *testing.B) { - var page strings.Builder - for i := range 50000 { - fmt.Fprintf(&page, "project-%d\n", i, i) - } - in := page.String() - reps := []replacement{ - {from: []byte(`href="/simple/`), to: []byte(`href="http://proxy/pypi/simple/`)}, - {from: []byte(`href='/simple/`), to: []byte(`href='http://proxy/pypi/simple/`)}, - } - b.SetBytes(int64(len(in))) - for b.Loop() { - _, _ = io.Copy(io.Discard, &replacingReader{src: io.NopCloser(strings.NewReader(in)), replacements: reps}) - } -} diff --git a/internal/handler/pypi_upload_time_test.go b/internal/handler/pypi_upload_time_test.go deleted file mode 100644 index f61a5f0..0000000 --- a/internal/handler/pypi_upload_time_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package handler - -import ( - "io" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/registries/fetch" -) - -// pypiUploadTimeSetup serves the JSON API for newpkg from releases, which the -// test may change, and counts its requests. cache_metadata is off, so every -// lookup reaches upstream. -func pypiUploadTimeSetup(t *testing.T) (h http.Handler, setReleases func(string), jsonRequests *atomic.Int64, fetcher *mockFetcher) { - t.Helper() - var mu sync.Mutex - releases := "" - jsonRequests = &atomic.Int64{} - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/pypi/newpkg/json" { - http.NotFound(w, r) - return - } - jsonRequests.Add(1) - mu.Lock() - defer mu.Unlock() - w.Header().Set("Content-Type", contentTypeJSON) - _, _ = io.WriteString(w, releases) - })) - t.Cleanup(upstream.Close) - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "7d"} - setReleases = func(s string) { - mu.Lock() - defer mu.Unlock() - releases = s - } - return (&PyPIHandler{proxy: proxy, upstreamURL: upstream.URL, downloadURL: upstream.URL, proxyURL: "http://localhost"}).Routes(), - setReleases, jsonRequests, fetcher -} - -func uploaded(ago time.Duration) string { - return time.Now().Add(-ago).Format(time.RFC3339) -} - -// TestPyPICachedDownloadUsesStoredUploadTime checks that once a release's -// upload time is stored, downloads of its cached files don't read the JSON -// API again. -func TestPyPICachedDownloadUsesStoredUploadTime(t *testing.T) { - h, setReleases, jsonRequests, fetcher := pypiUploadTimeSetup(t) - setReleases(`{"releases": {"1.0.0": [{"upload_time_iso_8601": "` + uploaded(30*24*time.Hour) + `"}]}}`) - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("sdist")), ContentType: "application/octet-stream"} - - for range 3 { - w := serveGet(t, h, "/packages/packages/ab/cd/ef0123456789/newpkg-1.0.0.tar.gz") - wantStatus(t, w, http.StatusOK, "") - if w.Body.String() != "sdist" { - t.Fatalf("body = %q", w.Body.String()) - } - } - if got := jsonRequests.Load(); got != 1 { - t.Errorf("JSON API requests = %d, want 1", got) - } -} - -// TestPyPIFileAddedToReleaseIsChecked checks that a file uploaded to a -// release after its time was stored is not let through on the stored time: -// it isn't cached, so the JSON API is read, the newer time is stored, and -// from then on the whole release is withheld until it is out of cooldown. -func TestPyPIFileAddedToReleaseIsChecked(t *testing.T) { - h, setReleases, jsonRequests, fetcher := pypiUploadTimeSetup(t) - sdist := `{"upload_time_iso_8601": "` + uploaded(30*24*time.Hour) + `"}` - setReleases(`{"releases": {"1.0.0": [` + sdist + `]}}`) - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("sdist")), ContentType: "application/octet-stream"} - - const ( - sdistPath = "/packages/packages/ab/cd/ef0123456789/newpkg-1.0.0.tar.gz" - wheelPath = "/packages/packages/ab/cd/ef0123456789/newpkg-1.0.0-py3-none-any.whl" - ) - wantStatus(t, serveGet(t, h, sdistPath), http.StatusOK, "") - - // A wheel is added to the release an hour ago. - setReleases(`{"releases": {"1.0.0": [` + sdist + `, {"upload_time_iso_8601": "` + uploaded(time.Hour) + `"}]}}`) - fetcher.fetchCalled = false - wantStatus(t, serveGet(t, h, wheelPath), http.StatusNotFound, "") - if fetcher.fetchCalled { - t.Error("fetched a file that is inside the cooldown window") - } - if got := jsonRequests.Load(); got != 2 { - t.Errorf("JSON API requests = %d, want 2 (the new file is looked up)", got) - } - - // The stored time is now the wheel's, so the cached sdist is withheld - // too, as the JSON API lookup would do, without asking it. - wantStatus(t, serveGet(t, h, sdistPath), http.StatusNotFound, "") - if got := jsonRequests.Load(); got != 2 { - t.Errorf("JSON API requests = %d, want 2", got) - } -} diff --git a/internal/handler/query_in_path_test.go b/internal/handler/query_in_path_test.go deleted file mode 100644 index d9f412f..0000000 --- a/internal/handler/query_in_path_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package handler - -import ( - "errors" - "net/http" - "net/http/httptest" - "sync/atomic" - "testing" -) - -// TestHandlersRejectQueryCharsInPath checks that handlers which build upstream -// URLs from the decoded path refuse an encoded '?' or '#' themselves, without -// relying on the server middleware, and never contact the upstream or fetch -// an artifact. -func TestHandlersRejectQueryCharsInPath(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - _, _ = w.Write([]byte("upstream")) - })) - defer upstream.Close() - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher.fetchErr = errors.New("artifact fetched") - - handlers := map[string]struct { - routes http.Handler - paths []string - }{ - "debian": {NewDebianHandler(proxy, testProxyURL, upstream.URL, nil).Routes(), []string{ - "/pool/main/h/hello/hello_2.10-3_amd64.deb%3F/foo_9.9_amd64.deb", - "/dists/trixie/InRelease%23x", - }}, - "rpm": {NewRPMHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes(), []string{ - "/Packages/n/nginx-1.24.0-2.fc39.x86_64.rpm%3F/foo-9.9-1.fc39.x86_64.rpm", - "/repodata/repomd.xml%3Fx", - }}, - "apk": {NewAPKHandler(proxy, testProxyURL, map[string]string{"alpine": upstream.URL}).Routes(), []string{ - "/alpine/v3.20/main/x86_64/busybox-1.36.1-r29.apk%3F/foo-9.9-r0.apk", - "/alpine/v3.20/main/x86_64/APKINDEX.tar.gz%23x", - }}, - "pypi": {NewPyPIHandlerWithUpstreams(proxy, testProxyURL, upstream.URL, upstream.URL).Routes(), []string{ - "/packages/aa/bb/requests-2.31.0.tar.gz%3F/foo-9.9.tar.gz", - "/simple/requests%3Fx/", - }}, - "cran": {NewCRANHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes(), []string{ - "/src/contrib/ggplot2_3.4.4.tar.gz%3Ffoo_9.9.tar.gz", - "/bin/macosx%23x/contrib/4.3/PACKAGES", - }}, - "maven": {NewMavenHandler(proxy, testProxyURL, upstream.URL, upstream.URL).Routes(), []string{ - "/com/example/lib/1.0/lib-1.0.jar%3F/com/example/foo/9.9/foo-9.9.jar", - "/com/example/lib/maven-metadata.xml%23x", - }}, - } - for name, tc := range handlers { - for _, target := range tc.paths { - w := httptest.NewRecorder() - tc.routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - if w.Code != http.StatusBadRequest { - t.Errorf("%s: GET %s = %d, want %d", name, target, w.Code, http.StatusBadRequest) - } - } - } - if got := requests.Load(); got != 0 { - t.Errorf("upstream requests = %d, want 0", got) - } - if fetcher.fetchCalled { - t.Errorf("artifact fetched from %s", fetcher.fetchedURL) - } -} diff --git a/internal/handler/read_metadata_test.go b/internal/handler/read_metadata_test.go index d87d416..b13bddb 100644 --- a/internal/handler/read_metadata_test.go +++ b/internal/handler/read_metadata_test.go @@ -3,12 +3,7 @@ package handler import ( "bytes" "errors" - "fmt" - "io" - "runtime" - "strings" "testing" - "testing/iotest" ) func TestReadMetadata(t *testing.T) { @@ -63,87 +58,3 @@ func TestReadMetadata(t *testing.T) { } }) } - -func TestReadMetadataSized(t *testing.T) { - const limit = 1024 - p := &Proxy{MetadataMaxSize: limit} - body := bytes.Repeat([]byte("x"), 100) - - for _, tt := range []struct { - name string - body []byte - size int64 - }{ - {"exact size", body, 100}, - {"unknown size", body, -1}, - {"size too small", body, 10}, - {"size too large", body, 900}, - {"size above limit", body, limit * 4}, - {"empty body", nil, 0}, - {"empty body with size", nil, 50}, - {"body at limit", bytes.Repeat([]byte("x"), limit), limit}, - {"body at limit, size too small", bytes.Repeat([]byte("x"), limit), 1}, - } { - t.Run(tt.name, func(t *testing.T) { - // OneByteReader makes sure short reads are not taken for EOF. - got, err := p.ReadMetadataSized(iotest.OneByteReader(bytes.NewReader(tt.body)), tt.size) - if err != nil { - t.Fatalf("ReadMetadataSized: %v", err) - } - if !bytes.Equal(got, tt.body) { - t.Errorf("got %d bytes, want %d", len(got), len(tt.body)) - } - }) - } - - for _, size := range []int64{-1, 10, limit, limit + 1, limit * 4} { - t.Run(fmt.Sprintf("over limit with size %d", size), func(t *testing.T) { - _, err := p.ReadMetadataSized(bytes.NewReader(make([]byte, limit+1)), size) - if !errors.Is(err, ErrMetadataTooLarge) { - t.Errorf("got error %v, want ErrMetadataTooLarge", err) - } - }) - } - - t.Run("read error", func(t *testing.T) { - errRead := errors.New("read failed") - r := io.MultiReader(bytes.NewReader(body), iotest.ErrReader(errRead)) - if _, err := p.ReadMetadataSized(r, 100); !errors.Is(err, errRead) { - t.Errorf("got error %v, want %v", err, errRead) - } - }) -} - -// TestReadMetadataSizedAllocatesOnce checks that a correct size hint reads -// into a single buffer of that size rather than a growing one. -func TestReadMetadataSizedAllocatesOnce(t *testing.T) { - const size = 8 << 20 - p := &Proxy{MetadataMaxSize: 64 << 20} - payload := bytes.Repeat([]byte("x"), size) - - var before, after runtime.MemStats - runtime.ReadMemStats(&before) - data, err := p.ReadMetadataSized(bytes.NewReader(payload), size) - runtime.ReadMemStats(&after) - if err != nil || len(data) != size { - t.Fatalf("read %d bytes, %v", len(data), err) - } - if allocated := after.TotalAlloc - before.TotalAlloc; allocated > size*3/2 { - t.Errorf("allocated %d bytes reading %d", allocated, size) - } -} - -// TestReadMetadataSizedTruncatedUpstream checks that an upstream body cut -// off before its Content-Length fails, while a body shorter than a size -// hint is still read. -func TestReadMetadataSizedTruncatedUpstream(t *testing.T) { - p := &Proxy{} - cut := io.MultiReader(strings.NewReader(`{"name":`), iotest.ErrReader(io.ErrUnexpectedEOF)) - if _, err := p.ReadMetadataSized(upstreamBodyReader{cut}, 100); !errors.Is(err, io.ErrUnexpectedEOF) { - t.Errorf("cut off body: error %v, want io.ErrUnexpectedEOF", err) - } - got, err := p.ReadMetadataSized(strings.NewReader(`{"name":"x"}`), 100) - if err != nil || string(got) != `{"name":"x"}` { - t.Errorf("short body = %q, %v", got, err) - } -} diff --git a/internal/handler/relay.go b/internal/handler/relay.go deleted file mode 100644 index 75b859a..0000000 --- a/internal/handler/relay.go +++ /dev/null @@ -1,110 +0,0 @@ -package handler - -import ( - "io" - "net/http" - "strings" - - "golang.org/x/net/http/httpguts" -) - -// relayResponse forwards an unmodified upstream body. Callers still own and -// close resp.Body, including when a failed transfer aborts the handler. A custom -// header copier may select or rewrite end-to-end headers; it only sees headers -// after hop-by-hop fields have been removed. -func (p *Proxy) relayResponse(w http.ResponseWriter, r *http.Request, resp *http.Response, copyHeaders func(http.Header, http.Header)) { - blocked := relayHopHeaders(resp.Header) - headers := resp.Header.Clone() - for name := range blocked { - headers.Del(name) - } - bodyAllowed := r.Method != http.MethodHead && resp.StatusCode >= http.StatusOK && - resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusResetContent && resp.StatusCode != http.StatusNotModified - if resp.StatusCode < http.StatusOK || resp.StatusCode == http.StatusNoContent { - headers.Del(headerContentLength) - } else if resp.StatusCode == http.StatusResetContent { - headers.Set(headerContentLength, "0") - } - if copyHeaders == nil { - copyHeaders = copyRelayHeaders - } - copyHeaders(w.Header(), headers) - - announced := make(map[string]bool) - if bodyAllowed { - for name := range resp.Trailer { - name = http.CanonicalHeaderKey(name) - if validRelayTrailer(name, blocked) { - announced[name] = true - w.Header().Add("Trailer", name) - } - } - if len(announced) > 0 { - // HTTP/1 trailers require chunking, not a fixed Content-Length. - w.Header().Del(headerContentLength) - } - } - w.WriteHeader(resp.StatusCode) - if !bodyAllowed || resp.Body == nil { - return - } - - written, err := io.Copy(w, resp.Body) - if err != nil { - upstreamURL := "" - if resp.Request != nil && resp.Request.URL != nil { - upstreamURL = resp.Request.URL.Redacted() - } - p.Logger.Warn("upstream response relay failed", "url", upstreamURL, - "status", resp.StatusCode, "bytes", written, "error", err) - // Headers are already committed: an error page would turn a truncated - // download into a seemingly successful response. Let net/http close the - // HTTP/1 connection or reset the HTTP/2 stream instead. - panic(http.ErrAbortHandler) - } - relayTrailers(w, resp.Trailer, announced, blocked) -} - -func copyRelayHeaders(dst, src http.Header) { - for name, values := range src { - for _, value := range values { - dst.Add(name, value) - } - } -} - -func relayHopHeaders(headers http.Header) map[string]bool { - blocked := map[string]bool{ - "Connection": true, "Proxy-Connection": true, "Keep-Alive": true, - "Proxy-Authenticate": true, "Proxy-Authorization": true, "Te": true, - "Trailer": true, "Transfer-Encoding": true, "Upgrade": true, - } - for _, value := range headers.Values("Connection") { - for _, name := range strings.Split(value, ",") { - if name = strings.TrimSpace(name); name != "" { - blocked[http.CanonicalHeaderKey(name)] = true - } - } - } - return blocked -} - -func validRelayTrailer(name string, blocked map[string]bool) bool { - return !blocked[name] && httpguts.ValidHeaderFieldName(name) && httpguts.ValidTrailerHeader(name) -} - -func relayTrailers(w http.ResponseWriter, trailers http.Header, announced, blocked map[string]bool) { - for name, values := range trailers { - name = http.CanonicalHeaderKey(name) - if !validRelayTrailer(name, blocked) { - continue - } - if !announced[name] { - // A trailer discovered only at EOF must not become a regular header - // or trigger automatic Content-Length on a short buffered response. - _ = http.NewResponseController(w).Flush() - name = http.TrailerPrefix + name - } - w.Header()[name] = append([]string(nil), values...) - } -} diff --git a/internal/handler/relay_test.go b/internal/handler/relay_test.go deleted file mode 100644 index 33db80b..0000000 --- a/internal/handler/relay_test.go +++ /dev/null @@ -1,275 +0,0 @@ -package handler - -import ( - "bytes" - "errors" - "fmt" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "strings" - "testing" - "testing/iotest" - - "github.com/git-pkgs/cooldown" - "github.com/go-chi/chi/v5/middleware" -) - -// Exercise the actual routes as well as the two shared entry points. Error -// responses also reach relay branches in handlers that normally parse metadata. -func relayTestRoutes(proxy *Proxy, upstream string) http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/upstream", func(w http.ResponseWriter, r *http.Request) { - proxy.ProxyUpstream(w, r, upstream, nil) - }) - mux.HandleFunc("/file", func(w http.ResponseWriter, r *http.Request) { - proxy.ProxyFile(w, r, upstream) - }) - mux.HandleFunc("/metadata", func(w http.ResponseWriter, r *http.Request) { - proxy.ProxyCached(w, r, upstream, "test", "index") - }) - const proxyURL = "http://proxy.local" - mount := func(prefix string, h http.Handler) { - mux.Handle(prefix+"/", http.StripPrefix(prefix, h)) - } - mount("/nuget", NewNuGetHandlerWithUpstreams(proxy, proxyURL, upstream, upstream).Routes()) - mount("/conan", NewConanHandlerWithUpstream(proxy, proxyURL, upstream).Routes()) - mount("/composer", NewComposerHandlerWithUpstreams(proxy, proxyURL, upstream, upstream).Routes()) - mount("/pypi", NewPyPIHandlerWithUpstreams(proxy, proxyURL, upstream, upstream).Routes()) - mount("/gem", NewGemHandlerWithUpstream(proxy, proxyURL, upstream).Routes()) - mount("/conda", NewCondaHandlerWithUpstream(proxy, proxyURL, upstream).Routes()) - mount("/hex", NewHexHandlerWithUpstreams(proxy, proxyURL, upstream, upstream).Routes()) - mount("/swift", NewSwiftHandler(proxy, proxyURL, upstream).Routes()) - mount("/v2", NewContainerHandlerWithRegistry(proxy, proxyURL, upstream).Routes()) - mount("/npm", NewNPMHandler(proxy, proxyURL, upstream).Routes()) - // Match the production recovery middleware: it must not swallow the abort. - return middleware.Recoverer(mux) -} - -func TestRelayRoutes(t *testing.T) { - for _, route := range []string{ - "/upstream", "/file", "/metadata", "/nuget/query", - "/conan/v2/conans/demo/1.0/user/stable/revisions/rev/files/other.txt", - "/composer/search.json", "/pypi/simple/", "/gem/api/v1/dependencies", - "/conda/conda-forge/noarch/repodata.json", - "/v2/library/demo/manifests/latest", "/v2/library/demo/tags/list", - "/npm/-/npm/v1/keys", - } { - t.Run(route, func(t *testing.T) { - for _, truncated := range []bool{false, true} { - t.Run(fmt.Sprintf("truncated=%v", truncated), func(t *testing.T) { - testRelayRoute(t, route, truncated) - }) - } - }) - } -} - -func testRelayRoute(t *testing.T, route string, truncated bool) { - t.Helper() - // Large enough to commit downstream headers before a body-copy failure. - body := strings.Repeat("x", 64*1024) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - conn, rw, err := w.(http.Hijacker).Hijack() - if err != nil { - t.Error(err) - return - } - defer func() { _ = conn.Close() }() - _, _ = fmt.Fprintf(rw, "HTTP/1.1 502 Bad Gateway\r\nContent-Type: text/plain\r\nConnection: X-Private\r\nX-Private: secret\r\nTransfer-Encoding: chunked\r\nTrailer: X-Checksum, X-Private\r\n\r\n%x\r\n%s\r\n", len(body), body) - if !truncated { - _, _ = fmt.Fprint(rw, "0\r\nX-Checksum: verified\r\nX-Late: discovered-at-eof\r\nX-Private: still-secret\r\n\r\n") - } - // The proxy may drop the connection before reading the whole - // response. What it made of the response under test is asserted - // downstream. - _ = rw.Flush() - })) - defer upstream.Close() - proxy := testProxy() - proxy.Logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - downstream := httptest.NewServer(relayTestRoutes(proxy, upstream.URL)) - defer downstream.Close() - - resp, err := downstream.Client().Get(downstream.URL + route) - if err != nil { - t.Fatal(err) - } - defer func() { _ = resp.Body.Close() }() - got, readErr := io.ReadAll(resp.Body) - if resp.StatusCode != http.StatusBadGateway { - t.Fatalf("status = %d, want 502", resp.StatusCode) - } - if resp.Header.Get("Connection") != "" || resp.Header.Get("X-Private") != "" || resp.Trailer.Get("X-Private") != "" { - t.Fatalf("connection-scoped fields leaked: headers=%v trailers=%v", resp.Header, resp.Trailer) - } - if truncated { - if readErr == nil { - t.Fatal("truncated upstream was delivered as a complete response") - } - return - } - if readErr != nil || string(got) != body { - t.Fatalf("body length = %d, want %d; error = %v", len(got), len(body), readErr) - } - if resp.Trailer.Get("X-Checksum") != "verified" || resp.Trailer.Get("X-Late") != "discovered-at-eof" { - t.Fatalf("trailers not relayed: %v", resp.Trailer) - } - if route == "/pypi/simple/" && !strings.Contains(resp.Header.Get("Vary"), "Accept") { - t.Error("PyPI lost Vary: Accept") - } -} - -func TestRelayResponseHeaders(t *testing.T) { - headers := http.Header{ - "Connection": {"keep-alive, x-private", " X-Second, Content-Length "}, - "X-Private": {"secret"}, "X-Second": {"secret"}, - "Keep-Alive": {"timeout=5"}, "Proxy-Connection": {"keep-alive"}, - "Proxy-Authenticate": {"challenge"}, "Proxy-Authorization": {"credentials"}, - "Te": {"trailers"}, "Trailer": {"X-Untrusted"}, - "Transfer-Encoding": {"chunked"}, "Upgrade": {"websocket"}, - "Content-Length": {"999"}, "Content-Type": {"application/octet-stream"}, - "Content-Encoding": {"gzip"}, "Etag": {`"v1"`}, - "Set-Cookie": {"a=1", "b=2"}, "Www-Authenticate": {"Bearer realm=test"}, - } - resp := &http.Response{StatusCode: http.StatusOK, Header: headers, Body: io.NopCloser(strings.NewReader("body"))} - w := httptest.NewRecorder() - w.Header().Set("X-Request-ID", "local") - testProxy().relayResponse(w, httptest.NewRequest(http.MethodGet, "/", nil), resp, nil) - for _, name := range []string{ - "Connection", "Keep-Alive", "Proxy-Connection", "Proxy-Authenticate", - "Proxy-Authorization", "Te", "Trailer", "Transfer-Encoding", "Upgrade", - "X-Private", "X-Second", "Content-Length", - } { - if got := w.Header().Get(name); got != "" { - t.Errorf("hop-by-hop header %s survived: %q", name, got) - } - } - for _, name := range []string{"Content-Type", "Content-Encoding", "Etag", "Set-Cookie", "Www-Authenticate"} { - if got := strings.Join(w.Header().Values(name), ","); got != strings.Join(headers.Values(name), ",") { - t.Errorf("end-to-end header %s changed: %q", name, got) - } - } - if w.Header().Get("X-Request-ID") != "local" || w.Body.String() != "body" { - t.Fatal("local header or response body changed") - } - if headers.Get("X-Private") != "secret" { - t.Fatal("relay mutated the upstream response headers") - } -} - -func TestRelayResponseBodiless(t *testing.T) { - for _, tt := range []struct { - method string - status int - length string - }{ - {http.MethodHead, http.StatusOK, "99"}, - {http.MethodGet, http.StatusNoContent, ""}, - {http.MethodGet, http.StatusResetContent, "0"}, - {http.MethodGet, http.StatusNotModified, "99"}, - {http.MethodGet, http.StatusEarlyHints, ""}, - } { - t.Run(fmt.Sprintf("%s/%d", tt.method, tt.status), func(t *testing.T) { - resp := &http.Response{ - StatusCode: tt.status, Header: http.Header{"Content-Length": {"99"}}, - Body: io.NopCloser(iotest.ErrReader(errors.New("body must not be read"))), - Trailer: http.Header{"X-Checksum": {"ignored"}}, - } - w := httptest.NewRecorder() - testProxy().relayResponse(w, httptest.NewRequest(tt.method, "/", nil), resp, nil) - if w.Code != tt.status || w.Body.Len() != 0 || w.Header().Get("Trailer") != "" { - t.Fatalf("unexpected bodiless response: %d %v %q", w.Code, w.Header(), w.Body.String()) - } - if got := w.Header().Get("Content-Length"); got != tt.length { - t.Errorf("Content-Length = %q, want %q", got, tt.length) - } - }) - } -} - -type relayFailWriter struct{ http.ResponseWriter } - -func (w relayFailWriter) Write([]byte) (int, error) { - return 0, errors.New("downstream disconnected") -} - -func TestRelayResponseCopyFailure(t *testing.T) { - for _, downstreamError := range []bool{false, true} { - t.Run(fmt.Sprintf("downstreamError=%v", downstreamError), func(t *testing.T) { - var logs bytes.Buffer - proxy := testProxy() - proxy.Logger = slog.New(slog.NewTextHandler(&logs, nil)) - resp := &http.Response{ - StatusCode: http.StatusOK, Header: make(http.Header), - Request: httptest.NewRequest(http.MethodGet, "http://upstream.test/file", nil), - Body: io.NopCloser(io.MultiReader(strings.NewReader("part"), iotest.ErrReader(io.ErrUnexpectedEOF))), - } - var w http.ResponseWriter = httptest.NewRecorder() - wantBytes := "bytes=4" - if downstreamError { - w = relayFailWriter{httptest.NewRecorder()} - wantBytes = "bytes=0" - } - defer func() { - if got := recover(); got != http.ErrAbortHandler { - t.Errorf("panic = %v, want http.ErrAbortHandler", got) - } - for _, field := range []string{"url=http://upstream.test/file", "status=200", wantBytes, "error="} { - if !strings.Contains(logs.String(), field) { - t.Errorf("log missing %q: %s", field, logs.String()) - } - } - }() - proxy.relayResponse(w, httptest.NewRequest(http.MethodGet, "/", nil), resp, nil) - }) - } -} - -func TestRelayResponseTrailerValidation(t *testing.T) { - resp := &http.Response{ - StatusCode: http.StatusOK, - Header: http.Header{"Connection": {"X-Private"}, "Content-Length": {"4"}}, - Body: io.NopCloser(strings.NewReader("body")), - Trailer: http.Header{ - "X-Checksum": {"verified"}, "X-Private": {"secret"}, - "Content-Length": {"999"}, "Content-Type": {"bad/type"}, - "Authorization": {"secret"}, "Transfer-Encoding": {"chunked"}, - "If-Match": {"secret"}, "Invalid Name": {"invalid"}, - }, - } - w := httptest.NewRecorder() - testProxy().relayResponse(w, httptest.NewRequest(http.MethodGet, "/", nil), resp, nil) - result := w.Result() - defer func() { _ = result.Body.Close() }() - if result.Header.Get("Content-Length") != "" { - t.Error("declared trailers must remove Content-Length") - } - if len(result.Trailer) != 1 || result.Trailer.Get("X-Checksum") != "verified" { - t.Fatalf("invalid trailers leaked: %v", result.Trailer) - } -} - -func TestRelayClosesBodyOnAbort(t *testing.T) { - for _, mode := range []string{"upstream", "file", "metadata"} { - t.Run(mode, func(t *testing.T) { - body := &closeTrackingReader{Reader: iotest.ErrReader(io.ErrUnexpectedEOF)} - proxy := testProxy() - proxy.HTTPClient = &http.Client{Transport: pypiRoundTripFunc(func(r *http.Request) (*http.Response, error) { - return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: body, Request: r}, nil - })} - defer func() { - if got := recover(); got != http.ErrAbortHandler { - t.Errorf("panic = %v, want http.ErrAbortHandler", got) - } - if !body.closed { - t.Error("upstream body was not closed on abort") - } - }() - relayTestRoutes(proxy, "http://upstream.test").ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/"+mode, nil)) - }) - } -} diff --git a/internal/handler/resolved_artifact_test.go b/internal/handler/resolved_artifact_test.go deleted file mode 100644 index 2a8f286..0000000 --- a/internal/handler/resolved_artifact_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "strings" - "sync" - "sync/atomic" - "testing" - - "github.com/git-pkgs/registries" - "github.com/git-pkgs/registries/client" - "github.com/git-pkgs/registries/fetch" -) - -type artifactTestRegistry struct { - calls atomic.Int64 - err error - artifacts []registries.Artifact -} - -func (*artifactTestRegistry) Ecosystem() string { return "maven" } -func (*artifactTestRegistry) URLs() client.URLBuilder { //nolint:ireturn // required by fetch.Registry - return &client.BaseURLs{} -} -func (r *artifactTestRegistry) FetchVersions(ctx context.Context, _ string) ([]registries.Version, error) { - r.calls.Add(1) - if err := ctx.Err(); err != nil { - return nil, err - } - if r.err != nil { - return nil, r.err - } - return []registries.Version{{Number: "1.0.0", Artifacts: r.artifacts}}, nil -} - -func TestGetOrFetchArtifactResolvedCacheHit(t *testing.T) { - p, db, store, fetcher := setupTestProxy(t) - fetcher.fetchErr = errors.New("unexpected artifact fetch on a cache hit") - seedPackage(t, db, store, "maven", "org.example:demo", "1.0.0", "demo-1.0.0.pom", "pom") - seedPackage(t, db, store, "maven", "org.example:demo", "1.0.0", "demo-1.0.0.jar", "jar") - registry := &artifactTestRegistry{artifacts: []registries.Artifact{{URL: "https://registry.example/demo-1.0.0.jar", Filename: "demo-1.0.0.jar"}}} - p.Resolver.RegisterRegistry(registry) - result, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "") - if err != nil { - t.Fatal(err) - } - defer func() { _ = result.Reader.Close() }() - body, err := io.ReadAll(result.Reader) - if err != nil || string(body) != "jar" || !result.Cached || result.Artifact.Filename != "demo-1.0.0.jar" { - t.Fatalf("result=%+v body=%q err=%v", result, body, err) - } - if fetcher.fetchCalled || registry.calls.Load() != 1 { - t.Fatalf("fetchCalled=%t resolutions=%d", fetcher.fetchCalled, registry.calls.Load()) - } -} - -func TestGetOrFetchArtifactResolvedMiss(t *testing.T) { - p, db, store, _ := setupTestProxy(t) - seedPackage(t, db, store, "maven", "org.example:demo", "1.0.0", "demo-1.0.0.pom", "pom") - registry := &artifactTestRegistry{artifacts: []registries.Artifact{{URL: "https://registry.example/demo-1.0.0.jar", Filename: "demo-1.0.0.jar"}}} - p.Resolver.RegisterRegistry(registry) - fetcher := &countingFetcher{content: "jar"} - p.Fetcher = fetcher - for i, cached := range []bool{false, true} { - result, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "") - if err != nil { - t.Fatal(err) - } - body, readErr := io.ReadAll(result.Reader) - _ = result.Reader.Close() - if readErr != nil || string(body) != "jar" || result.Cached != cached { - t.Fatalf("result=%+v body=%q err=%v; want cached=%t", result, body, readErr, cached) - } - if registry.calls.Load() != int64(i+1) { - t.Fatalf("resolutions=%d after request %d; want one per request", registry.calls.Load(), i+1) - } - } - if fetcher.calls.Load() != 1 || registry.calls.Load() != 2 { - t.Fatalf("downloads=%d resolutions=%d; want 1 and 2", fetcher.calls.Load(), registry.calls.Load()) - } -} - -func TestGetOrFetchArtifactNamedCacheHitDoesNotResolve(t *testing.T) { - p, db, store, _ := setupTestProxy(t) - seedPackage(t, db, store, "maven", "org.example:demo", "1.0.0", "demo-1.0.0.pom", "pom") - p.Resolver = nil // A named cache hit must not need a resolver. - result, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "demo-1.0.0.pom") - if err != nil { - t.Fatal(err) - } - defer func() { _ = result.Reader.Close() }() - if !result.Cached { - t.Fatal("expected cache hit") - } -} - -func TestGetOrFetchArtifactResolvedErrors(t *testing.T) { - for _, want := range []error{fetch.ErrNotFound, context.Canceled, errors.New("registry unavailable")} { - t.Run(want.Error(), func(t *testing.T) { - p, _, _, fetcher := setupTestProxy(t) - registry := &artifactTestRegistry{err: want} - p.Resolver.RegisterRegistry(registry) - _, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "") - if !errors.Is(err, want) || fetcher.fetchCalled { - t.Fatalf("err=%v fetched=%t", err, fetcher.fetchCalled) - } - if errors.Is(want, fetch.ErrNotFound) && !errors.Is(err, ErrUpstreamNotFound) { - t.Fatalf("missing upstream not-found classification: %v", err) - } - }) - } -} - -func TestGetOrFetchArtifactResolvedMissingStorage(t *testing.T) { - p, db, store, fetcher := setupTestProxy(t) - seedPackage(t, db, store, "npm", "demo", "1.0.0", "demo-1.0.0.tgz", "old") - clear(store.files) - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("new"))} - result, err := p.GetOrFetchArtifact(context.Background(), "npm", "demo", "1.0.0", "") - if err != nil { - t.Fatal(err) - } - defer func() { _ = result.Reader.Close() }() - if result.Cached || !fetcher.fetchCalled { - t.Fatalf("expected refetch: result=%+v fetched=%t", result, fetcher.fetchCalled) - } -} - -func TestGetOrFetchArtifactResolvedDenyBeforeResolution(t *testing.T) { - p, _, _, fetcher := setupTestProxy(t) - registry := &artifactTestRegistry{err: errors.New("must not resolve")} - p.Resolver.RegisterRegistry(registry) - setTestDenylist(t, p, "pkg:maven/org.example/demo@1.0.0") - _, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "") - if !errors.Is(err, ErrVersionDenied) || registry.calls.Load() != 0 || fetcher.fetchCalled { - t.Fatalf("err=%v resolutions=%d fetched=%t", err, registry.calls.Load(), fetcher.fetchCalled) - } -} - -func TestGetOrFetchArtifactResolvedConcurrentMisses(t *testing.T) { - p, _, _, _ := setupTestProxy(t) - fetcher := &countingFetcher{content: "tarball", delay: fetchHoldTime} - p.Fetcher = fetcher - start := make(chan struct{}) - var wg sync.WaitGroup - for range 8 { - wg.Go(func() { - <-start - result, err := p.GetOrFetchArtifact(context.Background(), "npm", "demo", "1.0.0", "") - if err != nil { - t.Error(err) - return - } - defer func() { _ = result.Reader.Close() }() - body, err := io.ReadAll(result.Reader) - if err != nil || string(body) != "tarball" { - t.Errorf("body=%q err=%v", body, err) - } - }) - } - close(start) - wg.Wait() - if fetcher.calls.Load() != 1 { - t.Fatalf("downloads=%d, want 1", fetcher.calls.Load()) - } -} - -func TestGetOrFetchArtifactResolvedWithoutFilename(t *testing.T) { - p, _, _, fetcher := setupTestProxy(t) - registry := &artifactTestRegistry{artifacts: []registries.Artifact{{URL: "https://registry.example/"}}} - p.Resolver.RegisterRegistry(registry) - _, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "") - if err == nil || !strings.Contains(err.Error(), "no filename") || fetcher.fetchCalled { - t.Fatalf("err=%v fetched=%t; expected rejection before download", err, fetcher.fetchCalled) - } -} - -func TestGetOrFetchArtifactResolvedInvalidCacheHash(t *testing.T) { - p, db, store, fetcher := setupTestProxy(t) - seedPackage(t, db, store, "npm", "demo", "1.0.0", "demo-1.0.0.tgz", "old") - art, err := db.GetArtifact("pkg:npm/demo@1.0.0", "demo-1.0.0.tgz") - if err != nil { - t.Fatal(err) - } - art.ContentHash.String = "invalid-hash" - if err := db.UpsertArtifact(art); err != nil { - t.Fatal(err) - } - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("new"))} - result, err := p.GetOrFetchArtifact(context.Background(), "npm", "demo", "1.0.0", "") - if err != nil { - t.Fatal(err) - } - defer func() { _ = result.Reader.Close() }() - body, err := io.ReadAll(result.Reader) - if err != nil || string(body) != "new" || result.Cached || !fetcher.fetchCalled { - t.Fatalf("result=%+v body=%q err=%v fetched=%t", result, body, err, fetcher.fetchCalled) - } -} diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go deleted file mode 100644 index a624dba..0000000 --- a/internal/handler/rewrite_cache.go +++ /dev/null @@ -1,662 +0,0 @@ -package handler - -import ( - "bytes" - "compress/gzip" - "container/list" - "context" - "crypto/rand" - "crypto/sha256" - "encoding/binary" - "encoding/hex" - "errors" - "io" - "net/http" - "slices" - "strconv" - "strings" - "sync" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/metrics" -) - -// rewriteCache keeps metadata documents after a handler has rewritten them. -// Rewriting means decoding the whole document into generic maps, changing its -// download URLs and encoding it again, which for a large npm packument or an -// expanded Composer document costs milliseconds to hundreds of milliseconds -// and megabytes of allocations. It ran on every request, cached metadata -// included. The cache keeps one rewrite per distinct upstream document, and -// concurrent requests for a document that is not cached yet share a single -// rewrite. -// -// Entries are keyed by a hash of the raw document, so new bytes from upstream -// are rewritten again rather than served stale. Everything else a rewrite -// depends on is fixed for the life of the process: the proxy URL is part of -// the key, and the denylist is loaded at startup. The exception is cooldown, -// which depends on the time of the request: with cooldown on, entries expire -// when the first version they held back leaves cooldown (see -// cooldown_expiry.go). -// -// Each entry keeps the ETag of its output, so clients can revalidate their -// copy without the output being hashed again on every request. Dates are not -// the cache's to give: they must agree across replicas, so the database holds -// them (see metadataDate). -// -// Each entry also keeps its output gzip-compressed, when that is smaller, so -// clients accepting gzip get it without the document being compressed again -// on every request, by the proxy or by a reverse proxy in front of it. -type rewriteCache struct { - maxBytes int64 - // now reads the time entries expire by; nil means time.Now. - now func() time.Time - - mu sync.Mutex - size int64 - order *list.List // most recently used at the front - entries map[string]*list.Element - inFlight map[string]*inflightRewrite -} - -type rewriteEntry struct { - key string - rewrittenMetadata - // expires is when the entry stops being served, or zero for never. - expires time.Time -} - -// size is how much of the cache's room the entry takes. -func (e *rewriteEntry) size() int64 { - return int64(len(e.plain) + len(e.gzipped)) -} - -// inflightRewrite is one rewrite that concurrent callers share. m and err are -// written before done closes and read only after, so the close is the -// handoff. -type inflightRewrite struct { - done chan struct{} - m rewrittenMetadata - err error -} - -// metadataValidators are what a client revalidates its copy of rewritten -// metadata with. lastModified is zero when there is none to give. -type metadataValidators struct { - etag string - lastModified time.Time -} - -// rewrittenMetadata is a rewritten metadata document with its validators. -// gzipped is the document gzip-compressed, or nil when there is no -// compressed copy to serve. The cache keeps only gzipped when there is one, -// so plain is nil then; content returns the document either way. -type rewrittenMetadata struct { - plain []byte - metadataValidators - gzipped []byte - // size is the length of the uncompressed document. - size int -} - -// content returns the document uncompressed. A cached document is kept -// only gzip-compressed, and is decompressed for each caller that needs the -// plain bytes: clients not accepting gzip, and handlers reading the document. -// The compressed copy was made in memory by gzipIfSmaller, so reading it back -// fails only on a bug; content returns nil then. -func (m rewrittenMetadata) content() []byte { - if m.plain != nil || m.gzipped == nil { - return m.plain - } - zr, err := gzip.NewReader(bytes.NewReader(m.gzipped)) - if err != nil { - return nil - } - out := bytes.NewBuffer(make([]byte, 0, m.size)) - if _, err := io.Copy(out, zr); err != nil { - return nil - } - return out.Bytes() -} - -// identity returns m to be served without its gzip copy. -func (m rewrittenMetadata) identity() rewrittenMetadata { - m.plain = m.content() - m.gzipped = nil - return m -} - -// compact returns m as the cache keeps it: only the gzip copy when there is -// one. -func (m rewrittenMetadata) compact() rewrittenMetadata { - if m.gzipped != nil { - m.plain = nil - } - return m -} - -// metadataETag returns the strong ETag of a rewritten document. -func metadataETag(out []byte) string { - sum := sha256.Sum256(out) - return `"` + hex.EncodeToString(sum[:]) + `"` -} - -// rewriteConfigNonce stands for the configuration a rewrite reads at startup, -// such as the denylist and cooldown rules, in ETags derived from a rewrite's -// inputs. It changes on every start, so no ETag outlives the configuration it -// was derived under. -var rewriteConfigNonce = rand.Text() - -// keptVersionsETag returns the strong ETag of a rewrite from what fixes its -// output: the upstream document's digest, the proxy URL and package name its -// download URLs carry, the versions cooldown and the denylist kept, and the -// startup configuration. Hashing these costs far less than hashing the -// rewritten document. Each field is length-prefixed, so different inputs never -// hash the same bytes. -func keptVersionsETag(digest, proxyURL, name string, kept []string) string { - h := sha256.New() - var length []byte - for _, field := range append([]string{rewriteConfigNonce, digest, proxyURL, name}, slices.Sorted(slices.Values(kept))...) { - length = binary.AppendUvarint(length[:0], uint64(len(field))) - _, _ = h.Write(length) - _, _ = io.WriteString(h, field) - } - return `"` + hex.EncodeToString(h.Sum(nil)) + `"` -} - -// keepingRewrite rewrites a metadata document and reports the versions it -// kept. Together with the document they fix the rewrite's output. -type keepingRewrite func([]byte) ([]byte, []string, error) - -// errSharedRewriteAbandoned is what waiters see if the caller running a -// shared rewrite panicked out of it. -var errSharedRewriteAbandoned = errors.New("shared metadata rewrite did not complete") - -// newRewriteCache returns a cache holding up to maxBytes of rewritten output. -// A size of zero or less disables it. -func newRewriteCache(maxBytes int64) *rewriteCache { - if maxBytes <= 0 { - return nil - } - return &rewriteCache{ - maxBytes: maxBytes, - order: list.New(), - entries: make(map[string]*list.Element), - inFlight: make(map[string]*inflightRewrite), - } -} - -// rewriteCacheKey identifies one rewrite: the ecosystem and proxy URL the -// handler rewrites for, the package, and the exact upstream bytes. -func rewriteCacheKey(ecosystem, proxyURL, name string, in []byte) string { - sum := sha256.Sum256(in) - return rewriteCacheKeyForDigest(ecosystem, proxyURL, name, "sha256:"+hex.EncodeToString(sum[:])) -} - -// rewriteCacheKeyForDigest is rewriteCacheKey for upstream bytes known only by -// their digest, in the form the metadata cache records it. -func rewriteCacheKeyForDigest(ecosystem, proxyURL, name, digest string) string { - return strings.Join([]string{ecosystem, proxyURL, name, digest}, "\x00") -} - -// get returns the cached rewrite for key, if there is one. -func (c *rewriteCache) get(key string) (rewrittenMetadata, bool) { - if c == nil { - return rewrittenMetadata{}, false - } - c.mu.Lock() - defer c.mu.Unlock() - e, ok := c.lookup(key, c.clock()) - if !ok { - return rewrittenMetadata{}, false - } - return e.rewrittenMetadata, true -} - -// lookup returns the live entry for key and marks it recently used. An -// expired entry is dropped. Called with mu held. -func (c *rewriteCache) lookup(key string, now time.Time) (*rewriteEntry, bool) { - el, ok := c.entries[key] - if !ok { - return nil, false - } - e := el.Value.(*rewriteEntry) - if !e.expires.IsZero() && !now.Before(e.expires) { - c.remove(el) - return nil, false - } - c.order.MoveToFront(el) - return e, true -} - -func (c *rewriteCache) clock() time.Time { - if c.now == nil { - return time.Now() - } - return c.now() -} - -// remove drops an entry. Called with mu held. -func (c *rewriteCache) remove(el *list.Element) { - e := el.Value.(*rewriteEntry) - c.order.Remove(el) - delete(c.entries, e.key) - c.size -= e.size() -} - -// rewrite returns rewrite(in) with its ETag and gzip copy, from the cache -// when it can. A ttl above zero makes the result expire after that long. -// Callers must treat the returned bytes as read-only: a cached result is -// shared. A caller waiting on another's rewrite leaves when ctx ends; the -// rewrite itself always runs to completion, so the result is cached for the -// next request. -func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, expiry func(start time.Time) time.Time, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { - if c == nil { - return rewriteWithETag(in, rewrite) - } - return c.produce(ctx, key, expiry, func() (rewrittenMetadata, error) { - m, err := rewriteWithETag(in, rewrite) - if err == nil { - m.gzipped = gzipIfSmaller(m.plain) - } - return m, err - }) -} - -// produce is rewrite for a document that build makes with its validators -// and gzip copy. -func (c *rewriteCache) produce(ctx context.Context, key string, expiry func(start time.Time) time.Time, build func() (rewrittenMetadata, error)) (rewrittenMetadata, error) { - - c.mu.Lock() - if e, ok := c.lookup(key, c.clock()); ok { - c.mu.Unlock() - return e.rewrittenMetadata, nil - } - if f, ok := c.inFlight[key]; ok { - c.mu.Unlock() - select { - case <-ctx.Done(): - return rewrittenMetadata{}, ctx.Err() - case <-f.done: - return f.m, f.err - } - } - f := &inflightRewrite{done: make(chan struct{}), err: errSharedRewriteAbandoned} - c.inFlight[key] = f - c.mu.Unlock() - - // expiry is given the time from before the rewrite, so an entry never - // outlives the time its cooldown decisions were made at. - start := c.clock() - defer func() { - c.mu.Lock() - delete(c.inFlight, key) - if f.err == nil { - var expires time.Time - if expiry != nil { - expires = expiry(start) - } - c.add(&rewriteEntry{key: key, rewrittenMetadata: f.m.compact(), expires: expires}) - } - c.mu.Unlock() - close(f.done) - }() - f.m, f.err = build() - return f.m, f.err -} - -// rewriteWithETag runs rewrite and computes the ETag of what it returns. -func rewriteWithETag(in []byte, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { - out, err := rewrite(in) - if err != nil { - return rewrittenMetadata{}, err - } - return rewrittenMetadata{plain: out, metadataValidators: metadataValidators{etag: metadataETag(out)}, size: len(out)}, nil -} - -// gzipIfSmaller returns body gzip-compressed, or nil when compressing does -// not make it smaller. -func gzipIfSmaller(body []byte) []byte { - var buf bytes.Buffer - zw := gzip.NewWriter(&buf) - if _, err := zw.Write(body); err != nil { - return nil - } - if err := zw.Close(); err != nil || buf.Len() >= len(body) { - return nil - } - return buf.Bytes() -} - -// add stores e and evicts least recently used entries until the cache is -// back under its limit. An entry larger than the whole cache is not stored. -// Called with mu held. -func (c *rewriteCache) add(e *rewriteEntry) { - n := e.size() - if n > c.maxBytes { - return - } - if old, ok := c.entries[e.key]; ok { - c.remove(old) - } - c.entries[e.key] = c.order.PushFront(e) - c.size += n - for c.size > c.maxBytes { - c.remove(c.order.Back()) - } -} - -// cachedRewrite rewrites a metadata document through the proxy's rewrite -// cache and returns it with its validators. digest identifies in when the -// metadata cache stored it, and is empty otherwise; when known it keys the -// cache, so the document isn't hashed again. -// -// With cooldown on, the cached rewrite expires when the first version it -// held back leaves cooldown, and after p.cooldownRewriteTTL at the latest. Its -// ETag is the hash of its output, and it is dated like any other, so clients -// revalidate with either validator and get the gzip copy. -// -// Without the rewrite cache, cooldown rewrites run on every request and go -// undated: when digest is known their ETag comes from their inputs rather -// than from hashing each output again, and that ETag changes with every -// restart, so replicas would never agree on a date for it. -func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name string, in []byte, digest string, rewrite keepingRewrite) (rewrittenMetadata, error) { - rewriteOnly := func(b []byte) ([]byte, error) { - out, _, err := rewrite(b) - return out, err - } - cooldown := p.cooldownEnabled() - - if p.rewrites != nil { - key := rewriteCacheKeyForDigest(ecosystem, proxyURL, name, digest) - if digest == "" { - key = rewriteCacheKey(ecosystem, proxyURL, name, in) - } - var expiry func(time.Time) time.Time - if cooldown { - // Recording runs from before the rewrite until after it, so it - // hears of every version the rewrite holds back. - rec := p.watchCooldown() - defer p.unwatchCooldown(rec) - expiry = func(start time.Time) time.Time { return p.cooldownExpiry(rec, start) } - } - m, err := p.rewrites.rewrite(ctx, key, in, expiry, rewriteOnly) - if err == nil { - m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) - } - return m, err - } - - if cooldown && digest != "" { - out, kept, err := rewrite(in) - if err != nil { - return rewrittenMetadata{}, err - } - return rewrittenMetadata{plain: out, metadataValidators: metadataValidators{etag: keptVersionsETag(digest, proxyURL, name, kept)}, size: len(out)}, nil - } - m, err := rewriteWithETag(in, rewriteOnly) - if err == nil && !cooldown { - m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) - } - return m, err -} - -// streamedRewrite rewrites a document stored in storage, through the -// proxy's rewrite cache, without holding the document or its rewrite in -// memory uncompressed. rewrite is given a way to open the stored document, -// as often as it needs to read it, and writes the rewritten document to w. -// The result is cached under the same key cachedRewrite uses for the -// document, and expires the same way with cooldown on. -// -// It returns errMetadataNotStored without the rewrite cache, which the -// result has to be kept in. -func (p *Proxy) streamedRewrite(ctx context.Context, ecosystem, proxyURL, name string, entry *database.MetadataCacheEntry, rewrite func(open func() (io.ReadCloser, error), w io.Writer) error) (rewrittenMetadata, error) { - if p.rewrites == nil { - return rewrittenMetadata{}, errMetadataNotStored - } - var expiry func(time.Time) time.Time - if p.cooldownEnabled() { - rec := p.watchCooldown() - defer p.unwatchCooldown(rec) - expiry = func(start time.Time) time.Time { return p.cooldownExpiry(rec, start) } - } - key := rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String) - m, err := p.rewrites.produce(ctx, key, expiry, func() (rewrittenMetadata, error) { - // The rewrite runs to completion for the cache even when the - // caller leaves. - readCtx := context.WithoutCancel(ctx) - return gzipRewrite(func(w io.Writer) error { - return rewrite(func() (io.ReadCloser, error) { return p.openStoredMetadata(readCtx, entry) }, w) - }) - }) - if err == nil { - m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) - } - return m, err -} - -// gzipRewrite runs write and keeps what it writes gzip-compressed only, with -// the ETag and length of the uncompressed output. -func gzipRewrite(write func(w io.Writer) error) (rewrittenMetadata, error) { - var buf bytes.Buffer - zw := gzip.NewWriter(&buf) - sum := sha256.New() - n := &countingWriter{w: io.Discard} - if err := write(io.MultiWriter(zw, sum, n)); err != nil { - return rewrittenMetadata{}, err - } - if err := zw.Close(); err != nil { - return rewrittenMetadata{}, err - } - return rewrittenMetadata{ - metadataValidators: metadataValidators{etag: `"` + hex.EncodeToString(sum.Sum(nil)) + `"`}, - gzipped: buf.Bytes(), - size: int(n.n), - }, nil -} - -// uncachedRewrite gives a rewrite made outside the cache its validators: an -// ETag of the output itself and, as cachedRewrite does, a date that replicas -// agree on. -func (p *Proxy) uncachedRewrite(ecosystem, proxyURL, name string, out []byte) rewrittenMetadata { - v := metadataValidators{etag: metadataETag(out)} - if !p.cooldownEnabled() { - v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) - } - return rewrittenMetadata{plain: out, metadataValidators: v, size: len(out)} -} - -func (p *Proxy) cooldownEnabled() bool { - return p.Cooldown != nil && p.Cooldown.Enabled() -} - -// storedRewrite returns the cached rewrite of the metadata stored for -// ecosystem and cacheKey, when that metadata is still within its TTL and a -// rewrite of it is cached and has not expired. It reads only the cache row, -// never the stored document, so a repeated request for a large packument -// costs a database lookup instead of reading and hashing the whole document -// again. When it reports false the caller takes the usual fetch and rewrite -// path. -func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewrittenMetadata, bool) { - m, _, ok := p.storedRewriteVariant(ecosystem, cacheKey, proxyURL, name, nil) - return m, ok -} - -// storedRewriteVariant is storedRewrite for a rewrite whose output depends on -// more than the stored document. variant is given the stored content type -// and returns the rest of what the rewrite depends on, as passed to -// variantDigest; nil means nothing else. It also returns the stored content -// type, as a fetch of the document would. -func (p *Proxy) storedRewriteVariant(ecosystem, cacheKey, proxyURL, name string, variant func(contentType string) []string) (rewrittenMetadata, string, bool) { - if p.rewrites == nil { - return rewrittenMetadata{}, "", false - } - if !p.CacheMetadata || p.DB == nil || p.MetadataTTL <= 0 { - return rewrittenMetadata{}, "", false - } - entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) - if err != nil || entry == nil || !entry.ContentDigest.Valid || entry.ContentEncoding.String != "" { - return rewrittenMetadata{}, "", false - } - if !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { - return rewrittenMetadata{}, "", false - } - contentType := contentTypeJSON - if entry.ContentType.Valid { - contentType = entry.ContentType.String - } - digest := entry.ContentDigest.String - if variant != nil { - digest = variantDigest(digest, nil, variant(contentType)...) - } - m, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, digest)) - if !ok { - return rewrittenMetadata{}, "", false - } - m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) - metrics.RecordMetadataCacheHit(ecosystem, m.size) - p.recordMetadataRead(entry) - return m, contentType, true -} - -// variantDigest returns a digest for a rewrite of in that also depends on -// parts, such as the content type or the versions a filter removes, so that -// rewrites differing in them are cached apart. digest is in's digest as the -// metadata cache records it, or empty to hash in. Passed to cachedRewrite in -// place of the document's digest, it keeps the metadata date keyed by the -// package alone. -func variantDigest(digest string, in []byte, parts ...string) string { - if digest == "" { - sum := sha256.Sum256(in) - digest = "sha256:" + hex.EncodeToString(sum[:]) - } - h := sha256.New() - var length []byte - for _, part := range parts { - length = binary.AppendUvarint(length[:0], uint64(len(part))) - _, _ = h.Write(length) - _, _ = io.WriteString(h, part) - } - return digest + "+" + hex.EncodeToString(h.Sum(nil)) -} - -// serveRewrittenMetadata writes rewritten metadata with its validators, or -// 304 Not Modified when they show the client's copy is current. A client -// accepting gzip gets the compressed copy when there is one. Being another -// representation, it has an ETag of its own. -func serveRewrittenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenMetadata) { - serveRewrittenMetadataAs(w, r, m, contentTypeJSON) -} - -// serveRewrittenMetadataAs is serveRewrittenMetadata for a document of -// another content type. -func serveRewrittenMetadataAs(w http.ResponseWriter, r *http.Request, m rewrittenMetadata, contentType string) { - etag := m.etag - compressed := m.gzipped != nil && acceptsGzip(r) - if compressed { - etag = gzipETag(m.etag) - } - ensureVary(w.Header(), headerAcceptEncoding) - w.Header().Set(headerETag, etag) - if !m.lastModified.IsZero() { - w.Header().Set(headerLastModified, m.lastModified.UTC().Format(http.TimeFormat)) - } - if notModified(r, etag, m.lastModified) { - w.WriteHeader(http.StatusNotModified) - return - } - w.Header().Set(headerContentType, contentType) - if compressed { - w.Header().Set(headerContentEncoding, "gzip") - w.WriteHeader(http.StatusOK) - _, _ = w.Write(m.gzipped) - return - } - if m.plain == nil && m.gzipped != nil { - // Decompressed as it is sent, rather than into a copy of the - // whole document. - zr, err := gzip.NewReader(bytes.NewReader(m.gzipped)) - if err != nil { - http.Error(w, "failed to read cached metadata", http.StatusInternalServerError) - return - } - w.Header().Set(headerContentLength, strconv.Itoa(m.size)) - w.WriteHeader(http.StatusOK) - _, _ = io.Copy(w, zr) - return - } - w.WriteHeader(http.StatusOK) - _, _ = w.Write(m.plain) -} - -// ensureVary adds field to the Vary header unless it is listed already. -func ensureVary(header http.Header, field string) { - for _, value := range header.Values("Vary") { - for listed := range strings.SplitSeq(value, ",") { - if strings.EqualFold(strings.TrimSpace(listed), field) { - return - } - } - } - header.Add("Vary", field) -} - -// gzipETag returns the ETag of the gzip representation of the document -// whose ETag is etag. -func gzipETag(etag string) string { - return strings.TrimSuffix(etag, `"`) + `-gzip"` -} - -// acceptsGzip reports whether the request's Accept-Encoding allows a gzip -// response: gzip or x-gzip, or else "*", with a quality above zero. -func acceptsGzip(r *http.Request) bool { - wildcard := false - for _, value := range r.Header.Values(headerAcceptEncoding) { - for coding := range strings.SplitSeq(value, ",") { - name, params, _ := strings.Cut(coding, ";") - switch strings.ToLower(strings.TrimSpace(name)) { - case "gzip", "x-gzip": - return codingQualityPositive(params) - case "*": - wildcard = codingQualityPositive(params) - } - } - } - return wildcard -} - -// codingQualityPositive reports whether an Accept-Encoding entry's -// parameters give it a quality above zero. No q parameter means 1. -func codingQualityPositive(params string) bool { - for param := range strings.SplitSeq(params, ";") { - key, value, _ := strings.Cut(param, "=") - if strings.TrimSpace(key) != "q" { - continue - } - q, err := strconv.ParseFloat(strings.TrimSpace(value), 64) - return err == nil && q > 0 - } - return true -} - -// notModified reports whether a conditional request's validators match. -// If-Modified-Since only counts when the request has no If-None-Match. -func notModified(r *http.Request, etag string, lastModified time.Time) bool { - if header := r.Header.Get("If-None-Match"); header != "" { - return ifNoneMatchHits(header, etag) - } - if lastModified.IsZero() { - return false - } - since, err := http.ParseTime(r.Header.Get("If-Modified-Since")) - return err == nil && !lastModified.Truncate(time.Second).After(since) -} - -// SetMetadataRewriteCacheSize enables the cache of rewritten metadata with -// room for maxBytes of output. Zero or less disables it. -func (p *Proxy) SetMetadataRewriteCacheSize(maxBytes int64) { - p.rewrites = newRewriteCache(maxBytes) - if p.rewrites != nil { - p.rewrites.now = p.clock - } -} diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go deleted file mode 100644 index 3f4c062..0000000 --- a/internal/handler/rewrite_cache_test.go +++ /dev/null @@ -1,321 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "errors" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// countingRewrite upper-cases its input and counts calls. If entered is set it -// is closed when the first call starts, and the call then waits on release. -type countingRewrite struct { - calls atomic.Int64 - entered chan struct{} - release chan struct{} - once sync.Once -} - -func (r *countingRewrite) fn(in []byte) ([]byte, error) { - r.calls.Add(1) - if r.entered != nil { - r.once.Do(func() { close(r.entered) }) - <-r.release - } - return []byte(strings.ToUpper(string(in))), nil -} - -func (r *countingRewrite) keeping(in []byte) ([]byte, []string, error) { - out, err := r.fn(in) - return out, nil, err -} - -func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { - c := newRewriteCache(1 << 20) - var r countingRewrite - key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte("doc")) - - for range 3 { - m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) - if err != nil || string(m.content()) != "DOC" { - t.Fatalf("rewrite = %q, %v", m.content(), err) - } - } - if got := r.calls.Load(); got != 1 { - t.Errorf("rewrites = %d, want 1", got) - } -} - -func TestRewriteCache_NewUpstreamBytesRewriteAgain(t *testing.T) { - c := newRewriteCache(1 << 20) - var r countingRewrite - - for _, doc := range []string{"v1", "v2"} { - key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte(doc)) - m, err := c.rewrite(context.Background(), key, []byte(doc), nil, r.fn) - if err != nil || string(m.content()) != strings.ToUpper(doc) { - t.Fatalf("rewrite(%s) = %q, %v", doc, m.content(), err) - } - } - if got := r.calls.Load(); got != 2 { - t.Errorf("rewrites = %d, want 2", got) - } -} - -// TestRewriteCache_ConcurrentRequestsShareRewrite asserts that requests -// arriving while a document is being rewritten wait for that rewrite instead -// of running their own. Callers that arrive after it finishes hit the cache, -// so the count holds however the goroutines are scheduled. -func TestRewriteCache_ConcurrentRequestsShareRewrite(t *testing.T) { - c := newRewriteCache(1 << 20) - r := countingRewrite{entered: make(chan struct{}), release: make(chan struct{})} - key := rewriteCacheKey("npm", "http://proxy", "typescript", []byte("doc")) - const n = 10 - - outs := make(chan string, n) - run := func() { - m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) - if err != nil { - t.Error(err) - } - outs <- string(m.content()) - } - go run() - <-r.entered - for range n - 1 { - go run() - } - close(r.release) - - for range n { - if out := <-outs; out != "DOC" { - t.Errorf("out = %q", out) - } - } - if got := r.calls.Load(); got != 1 { - t.Errorf("rewrites = %d, want 1", got) - } -} - -// TestRewriteCache_WaiterLeavingKeepsRewrite asserts that a caller waiting on -// another's rewrite returns its own context error when its client leaves, -// while the rewrite completes and is cached for the next request. -func TestRewriteCache_WaiterLeavingKeepsRewrite(t *testing.T) { - c := newRewriteCache(1 << 20) - r := countingRewrite{entered: make(chan struct{}), release: make(chan struct{})} - key := rewriteCacheKey("npm", "http://proxy", "typescript", []byte("doc")) - - first := make(chan error, 1) - go func() { - _, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) - first <- err - }() - <-r.entered - ctx, cancel := context.WithCancel(context.Background()) - cancel() - if _, err := c.rewrite(ctx, key, []byte("doc"), nil, r.fn); !errors.Is(err, context.Canceled) { - t.Errorf("waiter err = %v, want context.Canceled", err) - } - close(r.release) - if err := <-first; err != nil { - t.Fatal(err) - } - if m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn); err != nil || string(m.content()) != "DOC" { - t.Errorf("cached rewrite = %q, %v", m.content(), err) - } - if got := r.calls.Load(); got != 1 { - t.Errorf("rewrites = %d, want 1", got) - } -} - -func TestRewriteCache_EvictsLeastRecentlyUsed(t *testing.T) { - c := newRewriteCache(6) // room for two three-byte outputs - var r countingRewrite - keyFor := func(doc string) string { return rewriteCacheKey("npm", "http://proxy", doc, []byte(doc)) } - - for _, doc := range []string{"aaa", "bbb"} { - _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), nil, r.fn) - } - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), nil, r.fn) // aaa is now the most recent - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), nil, r.fn) // evicts bbb - - before := r.calls.Load() - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), nil, r.fn) - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), nil, r.fn) - if got := r.calls.Load() - before; got != 0 { - t.Errorf("aaa and ccc rewritten %d times, want both still cached", got) - } - _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), nil, r.fn) - if got := r.calls.Load() - before; got != 1 { - t.Errorf("bbb should have been evicted and rewritten") - } -} - -func TestRewriteCache_OutputLargerThanCacheIsNotStored(t *testing.T) { - c := newRewriteCache(2) - var r countingRewrite - key := rewriteCacheKey("npm", "http://proxy", "big", []byte("big")) - - _, _ = c.rewrite(context.Background(), key, []byte("big"), nil, r.fn) - if c.len() != 0 { - t.Errorf("cached %d entries, want 0", c.len()) - } -} - -func TestRewriteCache_ErrorsAreNotCached(t *testing.T) { - c := newRewriteCache(1 << 20) - var calls int - failing := func([]byte) ([]byte, error) { calls++; return nil, errors.New("bad document") } - key := rewriteCacheKey("npm", "http://proxy", "broken", []byte("doc")) - - for range 2 { - if _, err := c.rewrite(context.Background(), key, []byte("doc"), nil, failing); err == nil { - t.Fatal("expected the rewrite error") - } - } - if calls != 2 { - t.Errorf("rewrites = %d, want 2", calls) - } -} - -func TestCachedRewrite_DisabledRewritesEveryTime(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.SetMetadataRewriteCacheSize(0) - var r countingRewrite - - for range 2 { - _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) - } - if got := r.calls.Load(); got != 2 { - t.Errorf("rewrites = %d, want 2", got) - } -} - -// TestCachedRewrite_CooldownExpires asserts that with cooldown on a rewrite -// is cached for maxCooldownRewriteTTL only: cooldown filtering depends on the -// current time, so a cached rewrite would otherwise keep hiding a version -// after its cooldown ends. -func TestCachedRewrite_CooldownExpires(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} - proxy.now = clock.now - proxy.SetMetadataRewriteCacheSize(1 << 20) - proxy.Cooldown = &cooldown.Config{Default: "3d"} - var r countingRewrite - - for range 2 { - _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) - } - if got := r.calls.Load(); got != 1 { - t.Errorf("rewrites within the TTL = %d, want 1", got) - } - - clock.advance(maxCooldownRewriteTTL) - _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) - if got := r.calls.Load(); got != 2 { - t.Errorf("rewrites after the TTL = %d, want 2", got) - } -} - -// TestRewriteCacheExpiredEntryFreesRoom asserts that an expired entry is -// dropped when looked up, so its size no longer counts. -func TestRewriteCacheExpiredEntryFreesRoom(t *testing.T) { - c := newRewriteCache(1 << 20) - clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} - c.now = clock.now - upper := func(b []byte) ([]byte, error) { return bytes.ToUpper(b), nil } - if _, err := c.rewrite(context.Background(), "k", []byte("doc"), func(start time.Time) time.Time { return start.Add(time.Minute) }, upper); err != nil { - t.Fatal(err) - } - clock.advance(time.Minute) - if _, ok := c.get("k"); ok { - t.Fatal("expired entry was served") - } - if c.size != 0 || c.order.Len() != 0 { - t.Errorf("size %d, entries %d after expiry; want 0", c.size, c.order.Len()) - } -} - -// TestMetadataHandlers_ServeCachedRewrite asserts that repeated requests for -// the same npm or Composer metadata reuse one rewrite and return the same -// bytes, with the download URLs pointing at the proxy. -func TestMetadataHandlers_ServeCachedRewrite(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - switch r.URL.Path { - case "/left-pad": - _, _ = w.Write([]byte(`{"name":"left-pad","versions":{"1.3.0":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"}}}}`)) - case "/p2/vendor/pkg.json": - _, _ = w.Write([]byte(`{"packages":{"vendor/pkg":[{"version":"1.0.0","dist":{"type":"zip","url":"https://example.com/pkg-1.0.0.zip"}}]}}`)) - default: - http.NotFound(w, r) - } - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - proxy.SetMetadataRewriteCacheSize(1 << 20) - - cases := []struct { - name string - handler http.Handler - path string - }{ - {"npm", NewNPMHandler(proxy, "http://proxy.example", upstream.URL).Routes(), "/left-pad"}, - {"composer", NewComposerHandlerWithUpstreams(proxy, "http://proxy.example", upstream.URL, upstream.URL).Routes(), "/p2/vendor/pkg.json"}, - } - for i, c := range cases { - var bodies []string - for range 2 { - rec := httptest.NewRecorder() - c.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, c.path, nil)) - if rec.Code != http.StatusOK { - t.Fatalf("%s: status %d", c.name, rec.Code) - } - bodies = append(bodies, rec.Body.String()) - } - if bodies[0] != bodies[1] || !strings.Contains(bodies[0], "http://proxy.example") { - t.Errorf("%s: bodies %q and %q", c.name, bodies[0], bodies[1]) - } - if got := proxy.rewrites.len(); got != i+1 { - t.Errorf("%s: cached rewrites = %d, want %d", c.name, got, i+1) - } - } -} - -// len reports how many rewrites are cached. -func (c *rewriteCache) len() int { - if c == nil { - return 0 - } - c.mu.Lock() - defer c.mu.Unlock() - return len(c.entries) -} - -// TestCooldownRewriteTTL checks that the cap on a rewrite filtered by -// cooldown is the shortest cooldown, between a minute and a day. -func TestCooldownRewriteTTL(t *testing.T) { - for _, tc := range []struct { - shortest, want time.Duration - }{ - {0, 24 * time.Hour}, - {72 * time.Hour, 24 * time.Hour}, - {2 * time.Hour, 2 * time.Hour}, - {10 * time.Second, time.Minute}, - } { - p := &Proxy{ShortestCooldown: tc.shortest} - if got := p.cooldownRewriteTTL(); got != tc.want { - t.Errorf("shortest cooldown %v: TTL = %v, want %v", tc.shortest, got, tc.want) - } - } -} diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 62b2798..6440d0f 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -3,12 +3,8 @@ package handler import ( "fmt" "net/http" - pathpkg "path" "regexp" "strings" - - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/purl" ) const ( @@ -17,24 +13,6 @@ const ( rpmMatchCount = 5 // full match + name + version + release + arch ) -// Packages are stored as pkg:rpm/{name}, without a vendor and with the name's -// case kept. Package rules may name a vendor the way PURLs usually do, -// pkg:rpm/fedora/curl; it is dropped. -func init() { - retention.Default.Register(retention.Spec{Key: "rpm", CanonicalPackage: rpmRetentionKey}) -} - -// rpmRetentionKey maps a package rule to the stored package PURL. A name with -// a slash, or a namespace with more than one segment, can never match a stored -// package and is refused. -func rpmRetentionKey(p *purl.PURL) (string, bool) { - if p == nil || p.Type != "rpm" || strings.Contains(p.Namespace, "/") || strings.Contains(p.Name, "/") { - return "", false - } - canonical := purl.MakePURLString("rpm", p.Name, "") - return canonical, canonical != "" -} - // RPMHandler handles RPM/Yum repository protocol requests. // It proxies requests to upstream RPM repositories and caches .rpm packages. type RPMHandler struct { @@ -52,17 +30,10 @@ func NewRPMHandler(proxy *Proxy, proxyURL string) *RPMHandler { } } -// NewRPMHandlerWithUpstream creates an RPM handler with a custom upstream. -func NewRPMHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *RPMHandler { - h := NewRPMHandler(proxy, proxyURL) - h.upstreamURL = configuredUpstreamURL(upstreamURL, defaultRPMUpstream) - return h -} - // Routes returns the HTTP handler for RPM requests. // Mount this at /rpm on your router. func (h *RPMHandler) Routes() http.Handler { - return rejectQueryChars(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet && r.Method != http.MethodHead { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return @@ -79,31 +50,21 @@ func (h *RPMHandler) Routes() http.Handler { switch { case strings.HasSuffix(path, ".rpm"): // Package downloads - cache these - h.handlePackageDownload(w, r, cleanRPMPath(path)) + h.handlePackageDownload(w, r, path) case strings.Contains(path, "/repodata/"): - // Repository metadata + // Repository metadata - proxy without caching (changes frequently) h.handleMetadata(w, r, path) default: // Other files - proxy directly h.proxyFile(w, r, path) } - })) -} - -// cleanRPMPath drops empty and "." segments from a relative path, so each -// package has one cache key. ".." segments are rejected before this. -func cleanRPMPath(p string) string { - return strings.TrimPrefix(pathpkg.Clean("/"+p), "/") + }) } // handlePackageDownload fetches and caches .rpm packages. // Path format varies by repo structure, e.g.: // - releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm // - updates/39/Everything/x86_64/Packages/n/nginx-1.24.0-2.fc39.x86_64.rpm -// -// The cache key is the full path, since repositories under one upstream -// (Rocky and Alma on one mirror, say) can ship different bytes under one -// filename. func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Request, path string) { // Parse the path to extract package info name, version, arch := h.parseRPMPath(path) @@ -113,51 +74,29 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques return } + filename := path[strings.LastIndex(path, "/")+1:] downloadURL := fmt.Sprintf("%s/%s", h.upstreamURL, path) h.proxy.Logger.Info("rpm package download", - "name", name, "version", version, "arch", arch, "path", path) + "name", name, "version", version, "arch", arch, "filename", filename) result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), "rpm", name, version, path, downloadURL) + r.Context(), "rpm", name, version, filename, downloadURL) if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch package") + h.proxy.Logger.Error("failed to get rpm package", "error", err) + http.Error(w, "failed to fetch package", http.StatusBadGateway) return } - w.Header().Set(headerContentType, "application/x-rpm") - ServeArtifactRequest(w, r, result) + w.Header().Set("Content-Type", "application/x-rpm") + ServeArtifact(w, result) } -// handleMetadata proxies repository metadata files (repomd.xml, -// primary.xml.gz, etc.). repomd.xml changes with every repository update and -// follows the metadata TTL. The files it points at are usually named after -// their own checksum and so never change; those are kept for good. +// handleMetadata proxies repository metadata files (repomd.xml, primary.xml.gz, etc.). +// These change frequently so we don't cache them. func (h *RPMHandler) handleMetadata(w http.ResponseWriter, r *http.Request, path string) { - upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, path) - if rpmContentAddressed(path) { - h.proxy.proxyImmutable(w, r, upstreamURL, "rpm", h.metadataCacheKey(path), "*/*") - return - } - h.proxy.ProxyCached(w, r, upstreamURL, "rpm", h.metadataCacheKey(path), "*/*") -} - -// rpmChecksumPrefix matches a repodata filename that createrepo prefixed -// with the file's checksum: -primary.xml.gz. The hex lengths are those -// of the checksum types createrepo_c offers: SHA-1, SHA-224, SHA-256, -// SHA-384 and SHA-512. Repositories built with --simple-md-filenames use -// plain names, which can be replaced in place and follow the TTL. -var rpmChecksumPrefix = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{56}|[0-9a-f]{64}|[0-9a-f]{96}|[0-9a-f]{128})-[^/]+$`) - -// rpmContentAddressed reports whether a repodata path names a file after -// its checksum, so its content never changes. -func rpmContentAddressed(path string) bool { - return rpmChecksumPrefix.MatchString(path[strings.LastIndex(path, "/")+1:]) -} - -// metadataCacheKey is the metadata cache key of a repository path. -func (h *RPMHandler) metadataCacheKey(path string) string { - return hashedCacheKey(h.upstreamURL, path) + cacheKey := strings.ReplaceAll(path, "/", "_") + h.proxy.ProxyCached(w, r, fmt.Sprintf("%s/%s", h.upstreamURL, path), "rpm", cacheKey, "*/*") } // proxyFile proxies any file directly without caching. diff --git a/internal/handler/rpm_test.go b/internal/handler/rpm_test.go index d9561e3..851a304 100644 --- a/internal/handler/rpm_test.go +++ b/internal/handler/rpm_test.go @@ -1,14 +1,7 @@ package handler import ( - "io" - "net/http" - "net/http/httptest" - "sync/atomic" "testing" - - "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries/fetch" ) func TestRPMHandler_parseRPMPath(t *testing.T) { @@ -28,119 +21,3 @@ func TestRPMHandler_Routes(t *testing.T) { h := NewRPMHandler(nil, "http://localhost:8080") assertRoutesBasics(t, h.Routes(), "/repodata/repomd.xml", "/releases/../../../etc/passwd") } - -func TestRPMRetentionKey(t *testing.T) { - tests := []struct { - key string - want string - ok bool - }{ - {"pkg:rpm/curl", "pkg:rpm/curl", true}, - {"pkg:rpm/fedora/curl", "pkg:rpm/curl", true}, - {"pkg:rpm/Fedora/NetworkManager", "pkg:rpm/NetworkManager", true}, - {"pkg:rpm/fedora/libstdc++", "pkg:rpm/libstdc%2B%2B", true}, - {"pkg:rpm/fedora/40/curl", "", false}, - {"pkg:rpm/fedora/Packages%2Fcurl", "", false}, - {"pkg:deb/fedora/curl", "", false}, - } - for _, tt := range tests { - p, err := purl.Parse(tt.key) - if err != nil { - t.Fatalf("Parse(%s): %v", tt.key, err) - } - got, ok := rpmRetentionKey(p) - if ok != tt.ok || (ok && got != tt.want) { - t.Errorf("rpmRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) - } - } - if _, ok := rpmRetentionKey(nil); ok { - t.Error("rpmRetentionKey(nil) accepted") - } -} - -// TestRPMHandler_PackageDownloadCachesPerRepository checks that two -// repositories under one upstream that ship the same filename with different -// bytes are cached apart, as on a mirror root serving several distributions. -func TestRPMHandler_PackageDownloadCachesPerRepository(t *testing.T) { - packages := map[string]string{ - "/rocky/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm": "rocky bash", - "/almalinux/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm": "alma bash", - } - - var available atomic.Bool - available.Store(true) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - data, ok := packages[r.URL.Path] - if !available.Load() || !ok { - http.NotFound(w, r) - return - } - _, _ = w.Write([]byte(data)) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - srv := httptest.NewServer(NewRPMHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes()) - defer srv.Close() - - for round := range 2 { - for path, want := range packages { - resp, err := http.Get(srv.URL + path) - if err != nil { - t.Fatal(err) - } - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if resp.StatusCode != http.StatusOK || string(body) != want { - t.Errorf("round %d %s: %d %q, want 200 %q", round, path, resp.StatusCode, body, want) - } - } - // The second round must come from the cache. - available.Store(false) - } -} - -// TestRPMHandler_PackageDownloadCleansPath checks that spellings of one -// path with empty or "." segments share one cache entry. -func TestRPMHandler_PackageDownloadCleansPath(t *testing.T) { - const want = "bash" - var hits atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/rocky/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm" { - http.NotFound(w, r) - return - } - hits.Add(1) - _, _ = w.Write([]byte(want)) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) - srv := httptest.NewServer(NewRPMHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes()) - defer srv.Close() - - for _, path := range []string{ - "/rocky/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm", - "/rocky//9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm", - "/rocky/./9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm", - } { - resp, err := http.Get(srv.URL + path) - if err != nil { - t.Fatal(err) - } - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if resp.StatusCode != http.StatusOK || string(body) != want { - t.Errorf("%s: %d %q, want 200 %q", path, resp.StatusCode, body, want) - } - } - if n := hits.Load(); n != 1 { - t.Errorf("upstream fetched %d times, want 1", n) - } -} diff --git a/internal/handler/scan_test.go b/internal/handler/scan_test.go deleted file mode 100644 index 7480bad..0000000 --- a/internal/handler/scan_test.go +++ /dev/null @@ -1,310 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "errors" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/scanner" - "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries/fetch" -) - -// newTestScanServer returns an httptest.Server implementing the HTTPScanner -// notify contract, always replying with the given verdict. -func newTestScanServer(t testing.TB, allowed bool, reason string) *httptest.Server { - t.Helper() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var body map[string]any - if err := json.NewDecoder(r.Body).Decode(&body); err != nil { - t.Errorf("decode scan notify body: %v", err) - } - if body["fetch_url"] == "" || body["fetch_url"] == nil { - t.Error("scan notify body missing fetch_url") - } - _ = json.NewEncoder(w).Encode(map[string]any{"allowed": allowed, "reason": reason}) - })) - t.Cleanup(srv.Close) - return srv -} - -func newTestScanGroup(t testing.TB, scanURL string, failOpen bool) *scanner.Group { - t.Helper() - g, err := scanner.NewGroup(config.ScanningConfig{ - Enabled: true, - FailOpen: failOpen, - Timeout: "15s", - SigningKey: "test-signing-key", - Scanners: []config.ScannerConfig{ - {Name: "test-scanner", URL: scanURL, Mode: "block"}, - }, - }, slog.New(slog.NewTextHandler(io.Discard, nil))) - if err != nil { - t.Fatalf("scanner.NewGroup() error: %v", err) - } - return g -} - -func TestGetOrFetchArtifact_ScanAllowed(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - proxy.Scanners = newTestScanGroup(t, newTestScanServer(t, true, "").URL, false) - - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("clean content")), - ContentType: "application/gzip", - } - - result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "leftpad", "1.0.0", "leftpad-1.0.0.tgz") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer func() { _ = result.Reader.Close() }() - - body, _ := io.ReadAll(result.Reader) - if string(body) != "clean content" { - t.Errorf("body = %q, want %q", body, "clean content") - } - - cached, err := db.GetCachedArtifact( - purl.MakePURLString("npm", "leftpad", ""), purl.MakePURLString("npm", "leftpad", "1.0.0"), "leftpad-1.0.0.tgz") - if err != nil { - t.Fatalf("GetCachedArtifact() error: %v", err) - } - if cached == nil { - t.Error("expected allowed artifact to be committed to the cache database") - } - if len(store.files) == 0 { - t.Error("expected allowed artifact bytes to remain in storage") - } -} - -func TestGetOrFetchArtifact_ScanBlocked(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - proxy.Scanners = newTestScanGroup(t, newTestScanServer(t, false, "malware detected").URL, false) - - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("evil content")), - ContentType: "application/gzip", - } - - _, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "evilpkg", "1.0.0", "evilpkg-1.0.0.tgz") - if err == nil { - t.Fatal("expected error for blocked artifact") - } - if !errors.Is(err, ErrArtifactBlocked) { - t.Errorf("error = %v, want wrapped ErrArtifactBlocked", err) - } - if !strings.Contains(err.Error(), "malware detected") { - t.Errorf("error %q does not include scanner reason", err.Error()) - } - - cached, err := db.GetCachedArtifact( - purl.MakePURLString("npm", "evilpkg", ""), purl.MakePURLString("npm", "evilpkg", "1.0.0"), "evilpkg-1.0.0.tgz") - if err != nil { - t.Fatalf("GetCachedArtifact() error: %v", err) - } - if cached != nil { - t.Error("blocked artifact must never be committed to the cache database") - } - if len(store.files) != 0 { - t.Errorf("blocked artifact bytes must be deleted from storage, got %d files", len(store.files)) - } -} - -func TestGetOrFetchArtifact_BlockedDeleteSurvivesClientDisconnect(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - - const scanDelay = 150 * time.Millisecond - blockingSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(scanDelay) - _ = json.NewEncoder(w).Encode(map[string]any{"allowed": false, "reason": "malware detected"}) - })) - t.Cleanup(blockingSrv.Close) - proxy.Scanners = newTestScanGroup(t, blockingSrv.URL, false) - - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("evil content")), - ContentType: "application/gzip", - } - - // The client disconnects long before the (genuinely malicious) verdict - // comes back; cleanup of the blocked bytes must not be skipped just - // because the client is gone. - ctx, cancel := context.WithCancel(context.Background()) - time.AfterFunc(20*time.Millisecond, cancel) - - _, err := proxy.GetOrFetchArtifact(ctx, "npm", "evilpkg", "1.0.0", "evilpkg-1.0.0.tgz") - if err == nil { - t.Fatal("expected error for blocked artifact") - } - if !errors.Is(err, ErrArtifactBlocked) { - t.Errorf("error = %v, want wrapped ErrArtifactBlocked", err) - } - - cached, _ := db.GetCachedArtifact( - purl.MakePURLString("npm", "evilpkg", ""), purl.MakePURLString("npm", "evilpkg", "1.0.0"), "evilpkg-1.0.0.tgz") - if cached != nil { - t.Error("blocked artifact must never be committed to the cache database") - } - if len(store.files) != 0 { - t.Errorf("blocked artifact bytes must still be deleted even though the client disconnected mid-scan, got %d orphaned files", len(store.files)) - } -} - -func TestGetOrFetchArtifact_ScanErrorFailClosed(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - - brokenSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - t.Cleanup(brokenSrv.Close) - proxy.Scanners = newTestScanGroup(t, brokenSrv.URL, false) - - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("content")), - ContentType: "application/gzip", - } - - _, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "flaky", "1.0.0", "flaky-1.0.0.tgz") - if err == nil { - t.Fatal("expected error when scanner infrastructure fails") - } - if !errors.Is(err, ErrArtifactBlocked) { - t.Errorf("error = %v, want wrapped ErrArtifactBlocked (fail-closed default)", err) - } - if strings.Contains(err.Error(), brokenSrv.URL) { - t.Errorf("error %q leaks the internal scanner URL to the client-facing message", err.Error()) - } - if !strings.Contains(err.Error(), "scan could not be completed") { - t.Errorf("error %q does not use the generic infra-failure message", err.Error()) - } - - cached, _ := db.GetCachedArtifact( - purl.MakePURLString("npm", "flaky", ""), purl.MakePURLString("npm", "flaky", "1.0.0"), "flaky-1.0.0.tgz") - if cached != nil { - t.Error("artifact must not be committed when scanning fails fail-closed") - } - if len(store.files) != 0 { - t.Errorf("artifact bytes must be deleted on scan infra failure, got %d files", len(store.files)) - } -} - -func TestGetOrFetchArtifact_ScanErrorFailOpen(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - - brokenSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - t.Cleanup(brokenSrv.Close) - proxy.Scanners = newTestScanGroup(t, brokenSrv.URL, true) - - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("content")), - ContentType: "application/gzip", - } - - result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "flaky", "1.0.0", "flaky-1.0.0.tgz") - if err != nil { - t.Fatalf("unexpected error: %v (FailOpen must treat scanner infra failure as allowed)", err) - } - defer func() { _ = result.Reader.Close() }() - - cached, err := db.GetCachedArtifact( - purl.MakePURLString("npm", "flaky", ""), purl.MakePURLString("npm", "flaky", "1.0.0"), "flaky-1.0.0.tgz") - if err != nil { - t.Fatalf("GetCachedArtifact() error: %v", err) - } - if cached == nil { - t.Error("expected artifact to be committed to the cache when scanning fails fail-open") - } - if len(store.files) == 0 { - t.Error("expected artifact bytes to remain in storage when scanning fails fail-open") - } -} - -func TestGetOrFetchArtifact_ScanSurvivesClientDisconnect(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - - const scanDelay = 150 * time.Millisecond - slowSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(scanDelay) - _ = json.NewEncoder(w).Encode(map[string]any{"allowed": true}) - })) - t.Cleanup(slowSrv.Close) - proxy.Scanners = newTestScanGroup(t, slowSrv.URL, false) - - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("clean content")), - ContentType: "application/gzip", - } - - // Simulate a client that disconnects shortly after issuing the request: - // its context is cancelled well before the scanner replies, but the - // scan itself must run to completion rather than being torn down with - // it. - ctx, cancel := context.WithCancel(context.Background()) - time.AfterFunc(20*time.Millisecond, cancel) - - start := time.Now() - result, err := proxy.GetOrFetchArtifact(ctx, "npm", "leftpad", "1.0.0", "leftpad-1.0.0.tgz") - elapsed := time.Since(start) - if err != nil { - t.Fatalf("unexpected error: %v (a cancelled client context must not be mistaken for a scanner failure)", err) - } - defer func() { _ = result.Reader.Close() }() - - if elapsed < scanDelay { - t.Errorf("GetOrFetchArtifact returned after %v, want it to wait out the full scan (%v) despite client cancellation", elapsed, scanDelay) - } - - cached, err := db.GetCachedArtifact( - purl.MakePURLString("npm", "leftpad", ""), purl.MakePURLString("npm", "leftpad", "1.0.0"), "leftpad-1.0.0.tgz") - if err != nil { - t.Fatalf("GetCachedArtifact() error: %v", err) - } - if cached == nil { - t.Error("expected artifact to be committed to the cache; a client disconnect must not cause a false block") - } - if len(store.files) == 0 { - t.Error("expected artifact bytes to remain in storage; a client disconnect must not delete a legitimately allowed artifact") - } -} - -func TestGetOrFetchArtifact_ScanDisabledIsNoOp(t *testing.T) { - proxy, db, _, fetcher := setupTestProxy(t) - // proxy.Scanners left nil: scanning disabled. - - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("content")), - ContentType: "application/gzip", - } - - result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "plainpkg", "1.0.0", "plainpkg-1.0.0.tgz") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer func() { _ = result.Reader.Close() }() - - cached, err := db.GetCachedArtifact( - purl.MakePURLString("npm", "plainpkg", ""), purl.MakePURLString("npm", "plainpkg", "1.0.0"), "plainpkg-1.0.0.tgz") - if err != nil { - t.Fatalf("GetCachedArtifact() error: %v", err) - } - if cached == nil { - t.Error("expected artifact to be cached when scanning is disabled") - } -} diff --git a/internal/handler/scanfetch.go b/internal/handler/scanfetch.go deleted file mode 100644 index 832da9c..0000000 --- a/internal/handler/scanfetch.go +++ /dev/null @@ -1,89 +0,0 @@ -package handler - -import ( - "crypto/hmac" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "time" -) - -// scanFetchURL builds a short-lived, HMAC-signed URL for the internal -// /_internal/scan-fetch route, so an external scanner can pull the exact -// bytes staged at path without going through cooldown or the scan hook -// itself. This is generated the same way for every storage backend: it -// never depends on Storage.SignedURL, which not every backend implements. -func (p *Proxy) scanFetchURL(path string, ttl time.Duration) string { - exp := time.Now().Add(ttl).Unix() - return fmt.Sprintf("%s/_internal/scan-fetch?path=%s&exp=%d&sig=%s", - p.ScanFetchBaseURL, url.QueryEscape(path), exp, hmacHex(p.ScanSigningKey, path, exp)) -} - -func hmacHex(key []byte, path string, exp int64) string { - mac := hmac.New(sha256.New, key) - _, _ = fmt.Fprintf(mac, "%s|%d", path, exp) - return hex.EncodeToString(mac.Sum(nil)) -} - -// ServeScanFetch streams a storage object to a caller presenting a valid -// short-lived HMAC token, so external scanners can pull a staged artifact -// without going through cooldown or the scan hook themselves. This handler -// never calls GetOrFetchArtifact/fetchAndCache/storeArtifact — the -// separation from the normal request path is structural, not a -// conditional bypass flag. -// -// This route exists only for scanners configured under ScanningConfig; the -// URL is minted by scanFetchURL and passed as fetch_url in the scan notify -// request. It is not part of the public API and should be restricted to -// internal-network access at the ingress/network-policy layer — the HMAC -// scoping (one object, short TTL) limits what a leaked token can do, but -// isn't a substitute for network restriction. -// -// @Summary Fetch a staged artifact for scanning -// @Description Streams the exact bytes staged in storage for a pre-cache security scan. -// @Description Requires a short-lived HMAC-signed token minted by the proxy itself and -// @Description delivered via the fetch_url field of the scan notify request (see the -// @Description Artifact Scanning section of docs/configuration.md). Not part of the -// @Description public API; restrict access to the scanner network at the ingress layer. -// @Tags scanning -// @Produce application/octet-stream -// @Param path query string true "Storage path of the staged artifact" -// @Param exp query int true "Token expiry, Unix seconds" -// @Param sig query string true "HMAC-SHA256 signature over the string path|exp" -// @Success 200 {file} file -// @Failure 403 {string} string "invalid, expired, or tampered token" -// @Failure 404 {string} string "object not found in storage, or scanning is not configured" -// @Router /_internal/scan-fetch [get] -func (p *Proxy) ServeScanFetch(w http.ResponseWriter, r *http.Request) { - if p.Scanners == nil || !p.Scanners.Enabled() || len(p.ScanSigningKey) == 0 { - http.Error(w, "not found", http.StatusNotFound) - return - } - - path := r.URL.Query().Get("path") - exp, err := strconv.ParseInt(r.URL.Query().Get("exp"), 10, 64) - if err != nil || containsPathTraversal(path) || time.Now().Unix() > exp { - http.Error(w, "invalid or expired token", http.StatusForbidden) - return - } - - want := hmacHex(p.ScanSigningKey, path, exp) - if !hmac.Equal([]byte(r.URL.Query().Get("sig")), []byte(want)) { - http.Error(w, "invalid signature", http.StatusForbidden) - return - } - - reader, err := p.Storage.Open(r.Context(), path) - if err != nil { - http.Error(w, "not found", http.StatusNotFound) - return - } - defer func() { _ = reader.Close() }() - - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = io.Copy(w, reader) -} diff --git a/internal/handler/scanfetch_test.go b/internal/handler/scanfetch_test.go deleted file mode 100644 index 280f1a2..0000000 --- a/internal/handler/scanfetch_test.go +++ /dev/null @@ -1,153 +0,0 @@ -package handler - -import ( - "log/slog" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/scanner" -) - -// newEnabledScanGroup returns a scanner.Group that reports Enabled() true, -// so tests can exercise ServeScanFetch's normal signature-checking path -// rather than tripping its "scanning not configured" guard. -func newEnabledScanGroup(t testing.TB) *scanner.Group { - t.Helper() - g, err := scanner.NewGroup(config.ScanningConfig{ - Enabled: true, - Timeout: "15s", - SigningKey: "test-signing-key", - Scanners: []config.ScannerConfig{ - {Name: "test-scanner", URL: "http://localhost/scan", Mode: "block"}, - }, - }, slog.Default()) - if err != nil { - t.Fatalf("scanner.NewGroup() error: %v", err) - } - return g -} - -func TestServeScanFetch_ValidToken(t *testing.T) { - proxy, _, store, _ := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - proxy.Scanners = newEnabledScanGroup(t) - store.files["npm/lodash/4.17.21/lodash-4.17.21.tgz"] = []byte("artifact bytes") - - target := proxy.scanFetchURL("npm/lodash/4.17.21/lodash-4.17.21.tgz", time.Minute) - - req := httptest.NewRequest(http.MethodGet, target, nil) - w := httptest.NewRecorder() - proxy.ServeScanFetch(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) - } - if w.Body.String() != "artifact bytes" { - t.Errorf("body = %q, want %q", w.Body.String(), "artifact bytes") - } -} - -func TestServeScanFetch_Expired(t *testing.T) { - proxy, _, store, _ := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - proxy.Scanners = newEnabledScanGroup(t) - store.files["npm/lodash/4.17.21/lodash-4.17.21.tgz"] = []byte("artifact bytes") - - target := proxy.scanFetchURL("npm/lodash/4.17.21/lodash-4.17.21.tgz", -time.Minute) - - req := httptest.NewRequest(http.MethodGet, target, nil) - w := httptest.NewRecorder() - proxy.ServeScanFetch(w, req) - - if w.Code != http.StatusForbidden { - t.Errorf("status = %d, want 403", w.Code) - } -} - -func TestServeScanFetch_TamperedSignature(t *testing.T) { - proxy, _, store, _ := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - proxy.Scanners = newEnabledScanGroup(t) - store.files["npm/lodash/4.17.21/lodash-4.17.21.tgz"] = []byte("artifact bytes") - - target := proxy.scanFetchURL("npm/lodash/4.17.21/lodash-4.17.21.tgz", time.Minute) - tampered := strings.Replace(target, "sig=", "sig=deadbeef", 1) - - req := httptest.NewRequest(http.MethodGet, tampered, nil) - w := httptest.NewRecorder() - proxy.ServeScanFetch(w, req) - - if w.Code != http.StatusForbidden { - t.Errorf("status = %d, want 403", w.Code) - } -} - -func TestServeScanFetch_PathTraversal(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - proxy.Scanners = newEnabledScanGroup(t) - - target := proxy.scanFetchURL("../../etc/passwd", time.Minute) - - req := httptest.NewRequest(http.MethodGet, target, nil) - w := httptest.NewRecorder() - proxy.ServeScanFetch(w, req) - - if w.Code != http.StatusForbidden { - t.Errorf("status = %d, want 403", w.Code) - } -} - -func TestServeScanFetch_ScanningDisabled(t *testing.T) { - proxy, _, store, _ := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - // proxy.Scanners left nil: scanning disabled. - store.files["npm/lodash/4.17.21/lodash-4.17.21.tgz"] = []byte("artifact bytes") - - target := proxy.scanFetchURL("npm/lodash/4.17.21/lodash-4.17.21.tgz", time.Minute) - - req := httptest.NewRequest(http.MethodGet, target, nil) - w := httptest.NewRecorder() - proxy.ServeScanFetch(w, req) - - if w.Code != http.StatusNotFound { - t.Errorf("status = %d, want 404 when scanning is disabled", w.Code) - } -} - -func TestServeScanFetch_NoSigningKey(t *testing.T) { - proxy, _, store, _ := setupTestProxy(t) - // proxy.ScanSigningKey left empty. - proxy.Scanners = newEnabledScanGroup(t) - store.files["npm/lodash/4.17.21/lodash-4.17.21.tgz"] = []byte("artifact bytes") - - target := proxy.scanFetchURL("npm/lodash/4.17.21/lodash-4.17.21.tgz", time.Minute) - - req := httptest.NewRequest(http.MethodGet, target, nil) - w := httptest.NewRecorder() - proxy.ServeScanFetch(w, req) - - if w.Code != http.StatusNotFound { - t.Errorf("status = %d, want 404 when no signing key is configured", w.Code) - } -} - -func TestServeScanFetch_MissingObject(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - proxy.ScanSigningKey = []byte("test-signing-key") - proxy.Scanners = newEnabledScanGroup(t) - - target := proxy.scanFetchURL("npm/missing/1.0.0/missing-1.0.0.tgz", time.Minute) - - req := httptest.NewRequest(http.MethodGet, target, nil) - w := httptest.NewRecorder() - proxy.ServeScanFetch(w, req) - - if w.Code != http.StatusNotFound { - t.Errorf("status = %d, want 404", w.Code) - } -} diff --git a/internal/handler/shared_holds_test.go b/internal/handler/shared_holds_test.go deleted file mode 100644 index 5536c00..0000000 --- a/internal/handler/shared_holds_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "path/filepath" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/database" - upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" - "github.com/git-pkgs/proxy/internal/metrics" - registryclient "github.com/git-pkgs/registries/client" - "github.com/git-pkgs/registries/fetch" - "github.com/prometheus/client_golang/prometheus/testutil" -) - -// limitedUpstream answers every request with 429 and a Retry-After of ten -// minutes. Requests wait for release, which is open unless a test closes it -// later. -type limitedUpstream struct { - server *httptest.Server - calls atomic.Int32 - entered chan struct{} - release chan struct{} - released sync.Once -} - -func newLimitedUpstream(t *testing.T, blocked bool) *limitedUpstream { - u := &limitedUpstream{entered: make(chan struct{}, 8), release: make(chan struct{})} - if !blocked { - u.open() - } - t.Cleanup(u.open) - u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - u.calls.Add(1) - u.entered <- struct{}{} - <-u.release - w.Header().Set("Retry-After", "600") - w.WriteHeader(http.StatusTooManyRequests) - })) - t.Cleanup(u.server.Close) - return u -} - -func (u *limitedUpstream) open() { u.released.Do(func() { close(u.release) }) } - -func (u *limitedUpstream) artifactURL() string { return u.server.URL + "/left-pad-1.3.0.tgz" } - -// useSharedGate routes p's upstream requests through its own rate limit gate, -// sharing holds through db as a replica does. -func useSharedGate(p *Proxy, db *database.DB, u *limitedUpstream) *upstreamhttp.SharedRateLimitGate { - gate := upstreamhttp.NewSharedRateLimitGate(registryclient.NewRateLimitGate(u.server.Client().Transport), db, nil) - client := &http.Client{Transport: gate} - p.HTTPClient = client - p.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithBaseDelay(time.Millisecond)) - p.RateLimits = gate - return gate -} - -func fetchLimited(ctx context.Context, p *Proxy, u *limitedUpstream) error { - res, err := p.GetOrFetchArtifactFromURL(ctx, "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", u.artifactURL()) - if err == nil { - _ = res.Reader.Close() - } - return err -} - -// requireHeld checks that err is answered like any held upstream request, -// with 503 and a Retry-After. -func requireHeld(t *testing.T, name string, err error) { - t.Helper() - w := httptest.NewRecorder() - if !writeUpstreamHeld(w, err) { - t.Fatalf("%s: err = %v, want a held upstream", name, err) - } - if w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" { - t.Fatalf("%s: status %d, Retry-After %q; want 503 with Retry-After", name, w.Code, w.Header().Get("Retry-After")) - } -} - -// TestSharedHoldStopsOtherInstance checks that once one instance is rate -// limited, another sharing its database refuses requests to that host -// without sending them, until the hold ends. -func TestSharedHoldStopsOtherInstance(t *testing.T) { - path := filepath.Join(t.TempDir(), "shared.db") - dbA, err := database.Create(path) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = dbA.Close() }) - dbB, err := database.Open(path) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = dbB.Close() }) - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - store := newMockStorage() - a := NewProxy(dbA, store, &mockFetcher{}, fetch.NewResolver(), logger) - b := NewProxy(dbB, store, &mockFetcher{}, fetch.NewResolver(), logger) - - u := newLimitedUpstream(t, false) - useSharedGate(a, dbA, u) - gateB := useSharedGate(b, dbB, u) - gateB.Refresh = 0 - - if err := fetchLimited(context.Background(), a, u); err == nil { - t.Fatal("instance a: want the rate limit error") - } - if got := u.calls.Load(); got != 1 { - t.Fatalf("upstream requests after a = %d, want 1", got) - } - - requireHeld(t, "instance b artifact", fetchLimited(context.Background(), b, u)) - _, _, err = b.FetchOrCacheMetadata(context.Background(), "npm", "left-pad", u.server.URL+"/left-pad", "application/json") - if !errors.Is(err, registryclient.ErrUpstreamHeld) { - t.Errorf("instance b metadata: err = %v, want a held upstream", err) - } - if got := u.calls.Load(); got != 1 { - t.Errorf("upstream requests after b = %d, want 1", got) - } - - // The hold ends when the upstream said, and is then deleted. - holds, err := dbB.UpstreamHolds(time.Now().Add(11 * time.Minute)) - if err != nil || len(holds) != 0 { - t.Fatalf("holds after expiry = %v, %v; want none", holds, err) - } - if err := b.upstreamHeld(u.artifactURL()); err != nil { - t.Errorf("instance b after expiry: %v, want no hold", err) - } -} - -// TestFetchLeaseRateLimitedHolderStopsWaiters checks that instances waiting -// on a fetch that was rate limited answer with the holder's Retry-After -// instead of each sending one more request to the limited host. -func TestFetchLeaseRateLimitedHolderStopsWaiters(t *testing.T) { - a, b, db := sharedInstances(t) - u := newLimitedUpstream(t, true) - useSharedGate(a, db, u) - gateB := useSharedGate(b, db, u) - // b has read the shared holds just before, and would not read them - // again for an hour on its own. - gateB.Refresh = time.Hour - if err := b.upstreamHeld(u.artifactURL()); err != nil { - t.Fatal(err) - } - - waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) - resA := make(chan error, 1) - go func() { resA <- fetchLimited(context.Background(), a, u) }() - <-u.entered - resB := make(chan error, 1) - go func() { resB <- fetchLimited(context.Background(), b, u) }() - waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) - u.open() - - if err := <-resA; err == nil { - t.Fatal("instance a: want the rate limit error") - } - requireHeld(t, "instance b", <-resB) - if got := u.calls.Load(); got != 1 { - t.Errorf("upstream requests = %d, want 1", got) - } -} diff --git a/internal/handler/stale_cache_test.go b/internal/handler/stale_cache_test.go deleted file mode 100644 index cf2dd7b..0000000 --- a/internal/handler/stale_cache_test.go +++ /dev/null @@ -1,163 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "strings" - "testing" -) - -// These tests cover an upstream re-publishing a version: the cache holds one -// artifact and upstream now declares another digest for it. - -const ( - stalePkgPURL = "pkg:npm/pkg" - staleVersionPURL = "pkg:npm/pkg@1.0.0" - staleFilename = "pkg-1.0.0.tgz" - staleStoragePath = "npm/pkg/1.0.0/pkg-1.0.0.tgz" - staleURL = "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz" -) - -// seedCachedArtifact commits content the way a fetch does. -func seedCachedArtifact(t *testing.T, proxy *Proxy, store *mockStorage, content string) { - t.Helper() - ctx := context.Background() - if _, _, err := store.Store(ctx, staleStoragePath, strings.NewReader(content)); err != nil { - t.Fatalf("seeding storage: %v", err) - } - artifact := testArtifact(content, staleVersionPURL, staleFilename, "application/gzip") - if err := proxy.updateCacheDB("npm", "pkg", stalePkgPURL, staleURL, staleStoragePath, artifact, ""); err != nil { - t.Fatalf("seeding cache record: %v", err) - } -} - -// cachedDigest reports the digest the cache record holds, or "" without one. -func cachedDigest(t *testing.T, proxy *Proxy) string { - t.Helper() - record, err := proxy.DB.GetCachedArtifact(stalePkgPURL, staleVersionPURL, staleFilename) - if err != nil { - t.Fatalf("reading cache record: %v", err) - } - if record == nil { - return "" - } - return record.Artifact.Digest.Encoded() -} - -func bytesPresent(store *mockStorage) bool { - r, err := store.Open(context.Background(), staleStoragePath) - if err != nil { - return false - } - _ = r.Close() - return true -} - -func TestStaleCacheCheckHasNoSideEffects(t *testing.T) { - proxy, _, store, _ := setupTestProxy(t) - seedCachedArtifact(t, proxy, store, "old bytes") - - res, err := proxy.getCachedArtifactWithUpstreamHash(context.Background(), - stalePkgPURL, staleVersionPURL, staleFilename, sha256Hex("new bytes")) - if err != nil { - t.Fatalf("cache check failed: %v", err) - } - if res != nil { - drain(res) - t.Fatal("stale entry was served") - } - if got := cachedDigest(t, proxy); got != sha256Hex("old bytes") { - t.Errorf("record digest = %q, want the stale one kept: the check must not discard", got) - } - if !bytesPresent(store) { - t.Error("stale bytes were deleted by the check") - } -} - -func TestStaleCacheIsDiscardedBeforeTheFetch(t *testing.T) { - proxy, _, store, fetcher := setupTestProxy(t) - seedCachedArtifact(t, proxy, store, "old bytes") - boom := errors.New("upstream unavailable") - fetcher.fetchErr = boom - - _, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), - "npm", "pkg", "1.0.0", staleFilename, staleURL, "sha256:"+sha256Hex("new bytes")) - if !errors.Is(err, boom) { - t.Fatalf("got %v, want the fetch failure", err) - } - if got := cachedDigest(t, proxy); got != "" { - t.Errorf("stale record survived a failed refresh, digest = %q", got) - } - // A request that read the stale record may still be opening its bytes, - // so they are queued for deletion rather than deleted. - if !bytesPresent(store) { - t.Error("stale bytes were deleted while a reader may still open them") - } - assertQueuedForDeletion(t, proxy.DB, staleStoragePath) -} - -func TestStaleCacheIsReplacedByTheFetch(t *testing.T) { - proxy, _, store, fetcher := setupTestProxy(t) - seedCachedArtifact(t, proxy, store, "old bytes") - fetcher.artifact = artifactBody("new bytes") - upstream := sha256Hex("new bytes") - - res, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), - "npm", "pkg", "1.0.0", staleFilename, staleURL, "sha256:"+upstream) - if err != nil { - t.Fatalf("refresh failed: %v", err) - } - got, err := io.ReadAll(res.Reader) - _ = res.Reader.Close() - if err != nil || string(got) != "new bytes" { - t.Fatalf("got %q (err %v), want the refreshed bytes", got, err) - } - if !fetcher.fetchCalled { - t.Error("stale entry was served without a fetch") - } - if d := cachedDigest(t, proxy); d != upstream { - t.Errorf("record digest = %q, want %q", d, upstream) - } - - fetcher.fetchCalled = false - res, err = proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), - "npm", "pkg", "1.0.0", staleFilename, staleURL, "sha256:"+upstream) - if err != nil { - t.Fatalf("request after refresh failed: %v", err) - } - drain(res) - if fetcher.fetchCalled || !res.Cached { - t.Errorf("request after refresh: fetched=%v cached=%v, want served from cache", fetcher.fetchCalled, res.Cached) - } -} - -// TestLateLeaderKeepsRefreshedEntry is the race, at the point it would happen: -// a caller whose cache check saw a stale entry reaches the coalescing step -// after another caller's fetch replaced it. It must serve the replacement. -func TestLateLeaderKeepsRefreshedEntry(t *testing.T) { - proxy, _, store, fetcher := setupTestProxy(t) - seedCachedArtifact(t, proxy, store, "new bytes") - fetcher.fetchErr = errors.New("must not fetch") - upstream := sha256Hex("new bytes") - - res, err := proxy.coalescedFetchFromURL(context.Background(), - "npm", "pkg", "1.0.0", staleFilename, stalePkgPURL, staleVersionPURL, staleURL, nil, upstream, "") - if err != nil { - t.Fatalf("late leader failed: %v", err) - } - got, err := io.ReadAll(res.Reader) - _ = res.Reader.Close() - if err != nil || string(got) != "new bytes" { - t.Fatalf("got %q (err %v), want the refreshed bytes", got, err) - } - if fetcher.fetchCalled { - t.Error("refreshed entry was fetched again") - } - if d := cachedDigest(t, proxy); d != upstream { - t.Errorf("refreshed record was discarded, digest = %q", d) - } - if !bytesPresent(store) { - t.Error("refreshed bytes were deleted") - } -} diff --git a/internal/handler/stale_metadata_test.go b/internal/handler/stale_metadata_test.go deleted file mode 100644 index 9cd8e75..0000000 --- a/internal/handler/stale_metadata_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package handler - -import ( - "net/http" - "net/http/httptest" - "sync/atomic" - "testing" - "time" -) - -// upstreamThen serves body once and answers every later request with status. -func upstreamThen(t *testing.T, body string, status int) *httptest.Server { - t.Helper() - var requests atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if requests.Add(1) == 1 { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(body)) - return - } - w.WriteHeader(status) - })) - t.Cleanup(srv.Close) - return srv -} - -// staleProxy returns a proxy whose cached metadata expires right away. -func staleProxy(t *testing.T, upstream *httptest.Server) *Proxy { - t.Helper() - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Millisecond - proxy.HTTPClient = upstream.Client() - return proxy -} - -// requestTwice requests path, lets the cached copy expire, and requests it -// again. -func requestTwice(t *testing.T, h http.Handler, path string) *httptest.ResponseRecorder { - t.Helper() - first := httptest.NewRecorder() - h.ServeHTTP(first, httptest.NewRequest(http.MethodGet, path, nil)) - if first.Code != http.StatusOK { - t.Fatalf("first request: status = %d, body %s", first.Code, first.Body) - } - time.Sleep(5 * time.Millisecond) - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - return w -} - -// staleStatuses are upstream answers the metadata cache falls back on. -var staleStatuses = []int{http.StatusBadGateway, http.StatusNotFound, http.StatusUnauthorized, http.StatusForbidden} - -const staleComposerDocument = `{"packages":{"acme/widget":[{"name":"acme/widget","version":"1.0.0",` + - `"dist":{"type":"zip","url":"https://repo.packagist.org/files/acme/widget/1.0.0/widget.zip"}}]},"minified":"composer/2.0"}` - -func TestComposerStaleMetadataIsMarked(t *testing.T) { - for _, status := range staleStatuses { - t.Run(http.StatusText(status), func(t *testing.T) { - upstream := upstreamThen(t, staleComposerDocument, status) - h := NewComposerHandlerWithUpstreams(staleProxy(t, upstream), "http://proxy.test", upstream.URL, upstream.URL).Routes() - - w := requestTwice(t, h, "/p2/acme/widget.json") - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200 from the cache, body %s", w.Code, w.Body) - } - if got := w.Header().Get("Warning"); got != staleWarning { - t.Errorf("Warning = %q, want %q", got, staleWarning) - } - }) - } -} - -const staleChefUniverse = `{"apt":{"7.5.0":{"location_type":"supermarket",` + - `"location_path":"https://supermarket.chef.io/api/v1","download_url":` + - `"https://supermarket.chef.io/api/v1/cookbooks/apt/versions/7.5.0/download","dependencies":{}}}}` - -func TestChefStaleMetadataIsMarked(t *testing.T) { - for _, status := range staleStatuses { - t.Run(http.StatusText(status), func(t *testing.T) { - upstream := upstreamThen(t, staleChefUniverse, status) - h := NewChefHandler(staleProxy(t, upstream), chefTestProxyURL, upstream.URL).Routes() - - w := requestTwice(t, h, "/universe") - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200 from the cache, body %s", w.Code, w.Body) - } - if got := w.Header().Get("Warning"); got != staleWarning { - t.Errorf("Warning = %q, want %q", got, staleWarning) - } - }) - } -} diff --git a/internal/handler/stream_artifacts_test.go b/internal/handler/stream_artifacts_test.go deleted file mode 100644 index af0e3f1..0000000 --- a/internal/handler/stream_artifacts_test.go +++ /dev/null @@ -1,411 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/git-pkgs/artifacts" - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/registries/fetch" -) - -func newStreamingProxy(t *testing.T, body string) (*Proxy, *mockStorage, *mockFetcher) { - t.Helper() - proxy, _, store, fetcher := setupTestProxy(t) - proxy.StreamArtifacts = true - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader(body)), - Size: int64(len(body)), - ContentType: "application/gzip", - } - return proxy, store, fetcher -} - -func TestStreamArtifactsFromURLStreamsWithoutStoring(t *testing.T) { - proxy, store, fetcher := newStreamingProxy(t, "fetched content") - - result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "newpkg", "1.0.0", - "newpkg-1.0.0.tar.gz", "https://pypi.org/files/newpkg-1.0.0.tar.gz") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer func() { _ = result.Reader.Close() }() - - body, err := io.ReadAll(result.Reader) - if err != nil { - t.Fatalf("reading body: %v", err) - } - if string(body) != "fetched content" { - t.Errorf("body = %q, want %q", body, "fetched content") - } - if fetcher.fetchedURL != "https://pypi.org/files/newpkg-1.0.0.tar.gz" { - t.Errorf("fetched URL = %q", fetcher.fetchedURL) - } - if result.Cached { - t.Error("streaming result reported as cached") - } - if result.Artifact.Size != int64(len("fetched content")) { - t.Errorf("Size = %d, want the upstream size", result.Artifact.Size) - } - if result.Artifact.MediaType != "application/gzip" { - t.Errorf("MediaType = %q", result.Artifact.MediaType) - } - if len(store.files) != 0 { - t.Errorf("streaming stored %d files, want none", len(store.files)) - } - cached, err := proxy.DB.GetCachedArtifact("pkg:pypi/newpkg", "pkg:pypi/newpkg@1.0.0", "newpkg-1.0.0.tar.gz") - if err != nil { - t.Fatalf("GetCachedArtifact: %v", err) - } - if cached != nil { - t.Error("streaming recorded the artifact in the cache database") - } -} - -func TestStreamArtifactsGetOrFetchArtifactStreamsWithoutStoring(t *testing.T) { - proxy, store, fetcher := newStreamingProxy(t, "tarball data") - - result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "leftpad", testVersion100, "leftpad-1.0.0.tgz") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer func() { _ = result.Reader.Close() }() - - body, _ := io.ReadAll(result.Reader) - if string(body) != "tarball data" { - t.Errorf("body = %q", body) - } - if !fetcher.fetchCalled { - t.Error("upstream was not fetched") - } - if len(store.files) != 0 { - t.Errorf("streaming stored %d files, want none", len(store.files)) - } -} - -func TestStreamArtifactsIgnoresCachedArtifacts(t *testing.T) { - proxy, _, store, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("old bytes"))} - url := "https://pypi.org/files/newpkg-1.0.0.tar.gz" - - // Populate the cache in normal mode first. - result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz", url) - if err != nil { - t.Fatalf("priming cache: %v", err) - } - _ = result.Reader.Close() - if len(store.files) != 1 { - t.Fatalf("expected the cache to hold the artifact, got %d files", len(store.files)) - } - - proxy.StreamArtifacts = true - cached, err := proxy.GetCachedArtifact(context.Background(), "pypi", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz") - if err != nil || cached != nil { - t.Fatalf("GetCachedArtifact = %v, %v; want nil, nil while streaming", cached, err) - } - - fetcher.fetchCalled = false - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("new bytes"))} - result, err = proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz", url) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer func() { _ = result.Reader.Close() }() - body, _ := io.ReadAll(result.Reader) - if !fetcher.fetchCalled || string(body) != "new bytes" { - t.Errorf("streaming served %q (fetched=%v), want a fresh upstream fetch", body, fetcher.fetchCalled) - } -} - -func TestStreamArtifactsVerifiesUpstreamDigest(t *testing.T) { - const content = "blob bytes" - - t.Run("matching digest streams the body", func(t *testing.T) { - proxy, _, _ := newStreamingProxy(t, content) - result, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), "oci", "library/app", "v1", - "blob", "https://registry.test/blob", "sha256:"+sha256Hex(content)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer func() { _ = result.Reader.Close() }() - - body, err := io.ReadAll(result.Reader) - if err != nil { - t.Fatalf("reading verified body: %v", err) - } - if string(body) != content { - t.Errorf("body = %q", body) - } - if got := result.Artifact.Digest.Encoded(); got != sha256Hex(content) { - t.Errorf("Digest = %q, want the upstream digest", got) - } - if result.Artifact.Size >= 0 { - t.Errorf("Size = %d, want unknown so the response is chunked", result.Artifact.Size) - } - }) - - t.Run("mismatched digest fails the read", func(t *testing.T) { - proxy, _, _ := newStreamingProxy(t, content) - result, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), "oci", "library/app", "v1", - "blob", "https://registry.test/blob", "sha256:"+sha256Hex("other bytes")) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer func() { _ = result.Reader.Close() }() - - if _, err := io.ReadAll(result.Reader); !errors.Is(err, ErrArtifactDigestMismatch) { - t.Errorf("read error = %v, want ErrArtifactDigestMismatch", err) - } - }) -} - -func TestServeArtifactAbortsOnStreamedDigestMismatch(t *testing.T) { - proxy, _, _ := newStreamingProxy(t, "blob bytes") - result, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), "oci", "library/app", "v1", - "blob", "https://registry.test/blob", "sha256:"+sha256Hex("other bytes")) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - rec := httptest.NewRecorder() - defer func() { - if r := recover(); r != http.ErrAbortHandler { - t.Fatalf("recovered %v, want http.ErrAbortHandler", r) - } - if rec.Header().Get(headerContentLength) != "" { - t.Errorf("Content-Length = %q, want none so the client sees an unterminated response", - rec.Header().Get(headerContentLength)) - } - }() - ServeArtifact(rec, result) -} - -func TestNPMDownloadCooldownWhileStreaming(t *testing.T) { - now := time.Now() - packument := `{ - "name": "leftpad", - "dist-tags": {"latest": "2.0.0"}, - "time": { - "1.0.0": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `", - "2.0.0": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `" - }, - "versions": {"1.0.0": {}, "2.0.0": {}} - }` - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", contentTypeJSON) - _, _ = io.WriteString(w, packument) - })) - defer upstream.Close() - - tests := []struct { - name string - version string - wantStatus int - }{ - {"published before the window streams the tarball", testVersion100, http.StatusOK}, - {"published inside the window is withheld", "2.0.0", http.StatusNotFound}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proxy, store, fetcher := newStreamingProxy(t, "tarball data") - proxy.HTTPClient = upstream.Client() - proxy.Cooldown = &cooldown.Config{Default: "7d"} - - srv := httptest.NewServer(NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) - defer srv.Close() - - resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-" + tt.version + ".tgz") - if err != nil { - t.Fatalf("request failed: %v", err) - } - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - - if resp.StatusCode != tt.wantStatus { - t.Fatalf("status = %d, want %d", resp.StatusCode, tt.wantStatus) - } - if tt.wantStatus == http.StatusOK && string(body) != "tarball data" { - t.Errorf("body = %q", body) - } - if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled { - t.Error("fetched a version that is still inside the cooldown window") - } - if len(store.files) != 0 { - t.Errorf("streaming stored %d files, want none", len(store.files)) - } - }) - } -} - -// truncatingUpstream answers every request matching match with raw, then -// closes the connection, so the response body ends early. -func truncatingUpstream(t *testing.T, match func(*http.Request) bool, raw string, fallback http.HandlerFunc) *httptest.Server { - t.Helper() - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !match(r) { - fallback(w, r) - return - } - conn, buf, err := w.(http.Hijacker).Hijack() - if err != nil { - t.Errorf("hijack: %v", err) - return - } - _, _ = buf.WriteString(raw) - _ = buf.Flush() - _ = conn.Close() - })) - t.Cleanup(upstream.Close) - return upstream -} - -// requireIncompleteResponse fails unless reading the response surfaces an error, -// i.e. the client cannot mistake the body for a complete download. -func requireIncompleteResponse(t *testing.T, url string) { - t.Helper() - resp, err := http.Get(url) - if err != nil { - return - } - defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(resp.Body) - if err == nil { - t.Fatalf("client read a complete %d response (Content-Length %q, body %q), want an incomplete one", - resp.StatusCode, resp.Header.Get(headerContentLength), body) - } -} - -func useRealFetcher(t *testing.T, proxy *Proxy, upstream *httptest.Server) { - t.Helper() - proxy.HTTPClient = upstream.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - t.Cleanup(func() { _ = fetcher.Close() }) -} - -func TestStreamArtifactsOCIBlobShorterThanContentLengthIsIncomplete(t *testing.T) { - digest := "sha256:" + sha256Hex(strings.Repeat("x", 100)) - upstream := truncatingUpstream(t, - func(r *http.Request) bool { return strings.Contains(r.URL.Path, "/blobs/") }, - "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: 100\r\n\r\nshort", - http.NotFound) - - proxy, _, store, _ := setupTestProxy(t) - proxy.StreamArtifacts = true - useRealFetcher(t, proxy, upstream) - h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", "", map[string]string{"ghcr": upstream.URL}) - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - requireIncompleteResponse(t, srv.URL+"/upstream/ghcr/owner/demo/blobs/"+digest) - if len(store.files) != 0 { - t.Errorf("streaming stored %d files, want none", len(store.files)) - } -} - -func TestStreamArtifactsNPMTarballWithoutFinalChunkIsIncomplete(t *testing.T) { - upstream := truncatingUpstream(t, - func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, ".tgz") }, - "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nshort\r\n", - func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, `{"versions":{}}`) }) - - proxy, _, store, _ := setupTestProxy(t) - proxy.StreamArtifacts = true - useRealFetcher(t, proxy, upstream) - srv := httptest.NewServer(NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) - defer srv.Close() - - requireIncompleteResponse(t, srv.URL+"/leftpad/-/leftpad-1.0.0.tgz") - if len(store.files) != 0 { - t.Errorf("streaming stored %d files, want none", len(store.files)) - } -} - -func TestServeArtifactAbortsOnShortRead(t *testing.T) { - tests := []struct { - name string - reader io.Reader - size int64 - }{ - {"read error", io.MultiReader(strings.NewReader("short"), iotestErrReader{io.ErrUnexpectedEOF}), -1}, - {"fewer bytes than the declared size", strings.NewReader("short"), 100}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := &CacheResult{Reader: io.NopCloser(tt.reader), Artifact: artifacts.Artifact{Size: tt.size}} - defer func() { - if r := recover(); r != http.ErrAbortHandler { - t.Fatalf("recovered %v, want http.ErrAbortHandler", r) - } - }() - ServeArtifact(httptest.NewRecorder(), result) - }) - } -} - -type iotestErrReader struct{ err error } - -func (r iotestErrReader) Read([]byte) (int, error) { return 0, r.err } - -func TestStreamArtifactsSwiftArchiveHeadIgnoresCachedEntry(t *testing.T) { - archive := []byte("cached archive") - checksum := sha256.Sum256(archive) - var archiveRequests int - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasSuffix(r.URL.Path, ".zip") { - archiveRequests++ - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(checksum[:])) - })) - defer upstream.Close() - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader(string(archive))), - Size: int64(len(archive)), - ContentType: "application/zip", - } - packagePURL, versionPURL := packageurl.MakeCacheStrings("swift", "apple/example", "1.2.3") - cached, err := proxy.getOrFetchArtifactFromURLWithCachePURLs( - context.Background(), "swift", "apple/example", "1.2.3", "example-1.2.3.zip", - packagePURL, versionPURL, upstream.URL+"/apple/example/1.2.3.zip", nil, hex.EncodeToString(checksum[:]), "", - ) - if err != nil { - t.Fatalf("seeding cache in normal mode: %v", err) - } - _ = cached.Reader.Close() - - proxy.StreamArtifacts = true - fetcher.artifact = nil - fetcher.fetchErr = fetch.ErrNotFound - handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes() - - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/apple/example/1.2.3.zip", nil)) - if w.Code == http.StatusOK { - t.Fatalf("HEAD served the cached archive (Content-Length %q) while streaming artifacts", w.Header().Get(headerContentLength)) - } - if archiveRequests == 0 { - t.Error("HEAD did not ask the upstream archive") - } - - w = httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil)) - if w.Code == http.StatusOK { - t.Fatalf("GET served the cached archive while streaming artifacts") - } -} diff --git a/internal/handler/stream_miss.go b/internal/handler/stream_miss.go deleted file mode 100644 index fb8ae62..0000000 --- a/internal/handler/stream_miss.go +++ /dev/null @@ -1,458 +0,0 @@ -package handler - -import ( - "context" - "errors" - "fmt" - "io" - "strings" - "sync" - - "github.com/git-pkgs/artifacts" - "github.com/git-pkgs/proxy/internal/metrics" - "github.com/git-pkgs/registries/fetch" - "github.com/opencontainers/go-digest" -) - -// On a cache miss a marked request is sent the bytes as they arrive from -// upstream while they are stored, rather than after the whole artifact is -// stored. Clients with idle timeouts need this for large artifacts: git-lfs -// gives up after 30 seconds without data, pacman after 10. The caller that -// starts the shared fetch is handed the stream; callers joining it still wait -// for the stored artifact. - -type streamCacheMissKey struct{} - -// StreamCacheMiss marks ctx so that on an artifact cache miss the bytes are -// sent to the client while they are being stored. The server sets it on GET -// requests without a Range header; a range of a stream that is still -// arriving can't be served. -func StreamCacheMiss(ctx context.Context) context.Context { - return context.WithValue(ctx, streamCacheMissKey{}, true) -} - -func streamsCacheMiss(ctx context.Context) bool { - stream, _ := ctx.Value(streamCacheMissKey{}).(bool) - return stream && !servingHead(ctx) -} - -// missHandoff passes the client's stream from the shared fetch, which runs on -// its own goroutine, to the caller that started it. The caller can leave -// before a stream is offered, and the fetch must then not write to a stream -// nobody reads. -type missHandoff struct { - mu sync.Mutex - ready chan struct{} - stream *CacheResult - taken bool - abandoned bool - // feed is the stream's source. Set and read only on the fetch's - // goroutine. - feed *missFeed -} - -type missHandoffKey struct{} - -func handoffFrom(ctx context.Context) *missHandoff { - h, _ := ctx.Value(missHandoffKey{}).(*missHandoff) - return h -} - -// offer hands stream to the caller and reports whether it took it. -func (h *missHandoff) offer(stream *CacheResult) bool { - h.mu.Lock() - defer h.mu.Unlock() - if h.abandoned { - return false - } - h.stream, h.taken = stream, true - close(h.ready) - return true -} - -// holdUnrecorded gives a client that fell behind, and reads the stored copy -// of u, its own hold on u until it closes, so u isn't deleted under it. -func (h *missHandoff) holdUnrecorded(u *unrecordedObject) { - if h.feed != nil { - h.feed.holdUntilClose(u) - } -} - -// handedOver reports whether the caller took a stream. -func (h *missHandoff) handedOver() bool { - h.mu.Lock() - defer h.mu.Unlock() - return h.taken -} - -// runSharedFetchStreaming runs the shared fetch on its own goroutine and -// returns as soon as it offers a stream, or with its result if it finishes -// without one: on a recheck hit, an artifact another instance stored, an -// error, or an artifact a block-mode scanner has to see first. Waiters and -// the fetch lease still see the fetch end only once the artifact is stored. -func (p *Proxy) runSharedFetchStreaming(ctx context.Context, key string, f *inflightFetch, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { - h := &missHandoff{ready: make(chan struct{})} - type outcome struct { - res *CacheResult - err error - } - done := make(chan outcome, 1) - go func() { - res, err := p.runSharedFetch(context.WithValue(ctx, missHandoffKey{}, h), key, f, recheck, commit) - h.mu.Lock() - defer h.mu.Unlock() - if h.taken || h.abandoned { - // The caller has the stream, or has gone; nobody reads this. - if res != nil && res.Reader != nil { - _ = res.Reader.Close() - } - return - } - done <- outcome{res, err} - }() - - select { - case <-h.ready: - return h.stream, nil - case o := <-done: - return o.res, o.err - case <-ctx.Done(): - h.mu.Lock() - defer h.mu.Unlock() - if h.taken { - return h.stream, nil - } - h.abandoned = true - select { - case o := <-done: - if o.res != nil && o.res.Reader != nil { - _ = o.res.Reader.Close() - } - default: - } - return nil, ctx.Err() - } -} - -// storeFetched stores an artifact fetched from upstream. When the caller -// that started the fetch is waiting for a stream, it is sent the bytes as -// they are stored; see storeWhileServing. -func (p *Proxy) storeFetched(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared string, artifact *fetch.Artifact) (artifacts.Artifact, string, error) { - h := handoffFrom(ctx) - if h == nil || p.Scanners.Blocks(ecosystem) { - return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared, artifact) - } - return p.storeWhileServing(ctx, h, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared, artifact) -} - -// errNoReaderLeft stops reading upstream once neither storage nor the -// client takes the bytes any more. -var errNoReaderLeft = errors.New("neither storage nor the client is reading the artifact") - -// storeWhileServing copies the upstream body to storage and to the client at -// once, at the upstream's speed; see missFeed for a client that can't keep up. -// The client's copy is checked before its last bytes, so a mismatch aborts -// the response. A storage failure leaves the client's copy running. -func (p *Proxy) storeWhileServing(ctx context.Context, h *missHandoff, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared string, artifact *fetch.Artifact) (artifacts.Artifact, string, error) { - hash := strings.ToLower(upstreamHash) - checks, err := newIntegrityChecks(hash, declared) - if err != nil { - _ = artifact.Body.Close() - return artifacts.Artifact{}, "", err - } - - feed := newMissFeed(clientFeedLimit) - var client io.ReadCloser = feed - if hash != "" || declared != "" { - if client, err = checks.wrapFailOnMismatch(feed, func(reason string) { - p.Logger.Warn("artifact sent on a cache miss failed its integrity check; aborting the response", - "purl", versionPURL, "filename", filename, "reason", reason) - }); err != nil { - _ = artifact.Body.Close() - return artifacts.Artifact{}, "", err - } - } - stream := &CacheResult{ - Reader: client, - Artifact: artifacts.Artifact{ - PURL: versionPURL, - Filename: filename, - MediaType: artifact.ContentType, - // Unknown until stored, so the response goes out chunked and - // can still be aborted. - Size: -1, - }, - } - if hash != "" { - stream.Artifact.Digest = digest.Digest("sha256:" + hash) - } - if !h.offer(stream) { - _ = feed.Close() - return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared, artifact) - } - h.feed = feed - - storeR, storeW := io.Pipe() - type stored struct { - artifact artifacts.Artifact - path string - err error - } - storeDone := make(chan stored, 1) - go func() { - // Not cancelled with ctx: the client's request ends once it has the - // last byte, while storage may still be finishing the upload and the - // database write. Reading the upstream still stops with ctx, which - // ends the copy to storage too. - a, path, err := p.storeArtifact(context.WithoutCancel(ctx), ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared, - &fetch.Artifact{Body: storeR, Size: artifact.Size, ContentType: artifact.ContentType, ETag: artifact.ETag}) - // storeArtifact closes storeR; this covers a return before it read - // anything, so the copy below never blocks on it. - _ = storeR.Close() - storeDone <- stored{a, path, err} - }() - - readErr := copyToBoth(artifact.Body, storeW, feed) - _ = artifact.Body.Close() - _ = storeW.CloseWithError(readErr) - feed.end(readErr) - - out := <-storeDone - if feed.lagging() { - // Opened before returning, so an unrecorded object is still there; - // runSharedFetch then gives the client a hold on it. - feed.resolve(p.openStoredForFeed(ctx, out.artifact, out.path, out.err)) - } - if out.err != nil && readErr == nil && feed.fedFully() { - p.Logger.Error("artifact sent to the client but not cached", - "purl", versionPURL, "filename", filename, "url", upstreamURL, "error", out.err) - metrics.RecordArtifactServedUncached(ecosystem, metrics.FailureDetail{ - PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: out.err}) - } - return out.artifact, out.path, out.err -} - -// openStoredForFeed opens the stored copy for a client that fell behind. An -// unrecorded object is opened too: it is deleted only after this returns, and -// the client holds it from then on (missHandoff.holdUnrecorded). -func (p *Proxy) openStoredForFeed(ctx context.Context, a artifacts.Artifact, path string, storeErr error) (io.ReadCloser, error) { - if storeErr != nil && !errors.Is(storeErr, errUnrecorded) { - return nil, storeErr - } - res, err := p.openStoredArtifact(ctx, a, path) - if err != nil { - return nil, err - } - return res.Reader, nil -} - -// copyToBoth copies src to store and to the client's feed until src ends, -// dropping the store once a write to it fails. It returns the error that -// ended the copy, nil at the end of src. Feeding the client never blocks. -func copyToBoth(src io.Reader, store io.Writer, client *missFeed) error { - buffer := artifactCopyBufferPool.Get().(*[]byte) - defer artifactCopyBufferPool.Put(buffer) - buf := *buffer - storeOK := true - for { - n, err := src.Read(buf) - if n > 0 { - if storeOK { - if _, werr := store.Write(buf[:n]); werr != nil { - storeOK = false - } - } - // A client that has gone, or that waits for the stored copy, - // has no use for the rest once storing has failed. - if !client.push(buf[:n]) && !storeOK { - return errNoReaderLeft - } - } - if errors.Is(err, io.EOF) { - return nil - } - if err != nil { - return err - } - } -} - -// clientFeedLimit bounds the bytes held for a client that reads a streamed -// miss slower than they arrive. A variable so tests can lower it. -var clientFeedLimit = 8 << 20 - -// missFeed is the client's side of a streamed miss. The upstream read pushes -// into it without blocking, so a slow client can't hold back the copy to -// storage, the fetch lease or the callers waiting on the fetch. Once the -// client is more than the limit behind, it stops being fed: it reads what -// is held, then continues from the stored copy where it left off. -type missFeed struct { - mu sync.Mutex - cond *sync.Cond - chunks [][]byte - held int - limit int - // fed counts the bytes pushed before the client fell behind, where it - // continues in the stored copy. - fed int64 - ended bool - endErr error - behind bool - closed bool - early bool // closed before reading everything it was fed - - stored io.ReadCloser - storedErr error - resolved bool - skipped bool - // release drops the client's hold on an unrecorded stored copy. - release func() -} - -func newMissFeed(limit int) *missFeed { - m := &missFeed{limit: limit} - m.cond = sync.NewCond(&m.mu) - return m -} - -// push queues a copy of b and reports whether the client is still fed. -func (m *missFeed) push(b []byte) bool { - m.mu.Lock() - defer m.mu.Unlock() - if m.closed || m.behind { - return false - } - if m.held+len(b) > m.limit { - m.behind = true - return false - } - m.chunks = append(m.chunks, append([]byte(nil), b...)) - m.held += len(b) - m.fed += int64(len(b)) - m.cond.Broadcast() - return true -} - -// end records how the upstream read ended. -func (m *missFeed) end(err error) { - m.mu.Lock() - defer m.mu.Unlock() - m.ended, m.endErr = true, err - m.cond.Broadcast() -} - -func (m *missFeed) lagging() bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.behind && !m.closed -} - -// fedFully reports whether the client was sent every byte from upstream. -func (m *missFeed) fedFully() bool { - m.mu.Lock() - defer m.mu.Unlock() - return !m.behind && !m.early -} - -// resolve gives a client that fell behind the stored copy, or the error -// that left none. -func (m *missFeed) resolve(r io.ReadCloser, err error) { - m.mu.Lock() - defer m.mu.Unlock() - if m.closed { - if r != nil { - _ = r.Close() - } - return - } - m.stored, m.storedErr, m.resolved = r, err, true - m.cond.Broadcast() -} - -// holdUntilClose takes a hold on u for a client reading its stored copy and -// releases it when the client closes. -func (m *missFeed) holdUntilClose(u *unrecordedObject) { - m.mu.Lock() - defer m.mu.Unlock() - if m.closed || m.stored == nil { - return - } - u.refs.Add(1) - m.release = u.release -} - -func (m *missFeed) Read(p []byte) (int, error) { - m.mu.Lock() - for { - switch { - case m.closed: - m.mu.Unlock() - return 0, io.ErrClosedPipe - case len(m.chunks) > 0: - n := copy(p, m.chunks[0]) - if m.chunks[0] = m.chunks[0][n:]; len(m.chunks[0]) == 0 { - m.chunks[0] = nil - m.chunks = m.chunks[1:] - } - m.held -= n - m.mu.Unlock() - return n, nil - case m.behind && m.resolved: - r, err, skipped := m.stored, m.storedErr, m.skipped - m.skipped = true - m.mu.Unlock() - if err != nil { - return 0, err - } - if !skipped { - if err := skipTo(r, m.fed); err != nil { - return 0, err - } - } - return r.Read(p) - case !m.behind && m.ended: - err := m.endErr - m.mu.Unlock() - if err == nil { - err = io.EOF - } - return 0, err - } - m.cond.Wait() - } -} - -func (m *missFeed) Close() error { - m.mu.Lock() - if m.closed { - m.mu.Unlock() - return nil - } - m.closed = true - m.early = !m.ended || len(m.chunks) > 0 - m.chunks, m.held = nil, 0 - if m.stored != nil { - _ = m.stored.Close() - } - release := m.release - m.release = nil - m.cond.Broadcast() - m.mu.Unlock() - if release != nil { - // Outside the lock: the last release deletes the object. - release() - } - return nil -} - -// skipTo moves r past the first n bytes. -func skipTo(r io.Reader, n int64) error { - if s, ok := r.(io.Seeker); ok { - _, err := s.Seek(n, io.SeekStart) - return err - } - if _, err := io.CopyN(io.Discard, r, n); err != nil { - return fmt.Errorf("skipping to byte %d of the stored artifact: %w", n, err) - } - return nil -} diff --git a/internal/handler/stream_miss_test.go b/internal/handler/stream_miss_test.go deleted file mode 100644 index 2746588..0000000 --- a/internal/handler/stream_miss_test.go +++ /dev/null @@ -1,583 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/metrics" - "github.com/git-pkgs/proxy/internal/scanner" - "github.com/git-pkgs/registries/fetch" - "github.com/prometheus/client_golang/prometheus/testutil" -) - -// lockedBuffer is a log destination safe for the goroutine that stores a -// streamed artifact to write to while a test reads it. -type lockedBuffer struct { - mu sync.Mutex - buf bytes.Buffer -} - -func (b *lockedBuffer) Write(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.Write(p) -} - -func (b *lockedBuffer) String() string { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.String() -} - -// streamMissSetup is a Git LFS upstream whose object store sends the first -// half of an object, then waits for release before sending the rest, and a -// proxy serving it to clients the way the server marks their requests. -type streamMissSetup struct { - proxy *Proxy - store *mockStorage - client *httptest.Server - git *httptest.Server - objects *httptest.Server - release chan struct{} - content string // what the object store sends - oid string // the object ID clients ask for - path string - downloads atomic.Int32 - logs lockedBuffer -} - -// streamMissContent spans many copy buffers: a stream checked against its -// digest holds back its latest chunk until the next one arrives. -var streamMissContent = strings.Repeat("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\n", 400<<10/63) - -// streamMissPrefix is how much a client must get while the upstream still -// holds back the second half. -const streamMissPrefix = 64 << 10 - -func newStreamMiss(t *testing.T, served string) *streamMissSetup { - t.Helper() - return newStreamMissMarked(t, served, true) -} - -// newStreamMissMarked is newStreamMiss, with requests marked to detach -// their upstream fetch as the server does, or not, as when the server has -// no detached fetch to spare. -func newStreamMissMarked(t *testing.T, served string, detached bool) *streamMissSetup { - t.Helper() - sum := sha256.Sum256([]byte(streamMissContent)) - s := &streamMissSetup{content: served, oid: hex.EncodeToString(sum[:]), release: make(chan struct{})} - s.objects = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - s.downloads.Add(1) - half := len(s.content) / 2 - _, _ = io.WriteString(w, s.content[:half]) - w.(http.Flusher).Flush() - select { - case <-s.release: - case <-r.Context().Done(): - return - } - _, _ = io.WriteString(w, s.content[half:]) - })) - s.git = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var req lfsBatchRequest - _ = json.NewDecoder(r.Body).Decode(&req) - resp := lfsBatchResponse{Transfer: "basic"} - for _, o := range req.Objects { - resp.Objects = append(resp.Objects, lfsObjectResponse{OID: o.OID, Size: o.Size, - Actions: map[string]lfsAction{"download": {Href: s.objects.URL + "/objects/" + o.OID}}}) - } - w.Header().Set("Content-Type", gitLFSMediaType) - _ = json.NewEncoder(w).Encode(resp) - })) - - proxy, _, store, _ := setupTestProxy(t) - store.seekable = true - proxy.Logger = slog.New(slog.NewTextHandler(&s.logs, nil)) - proxy.HTTPClient = s.git.Client() - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(s.objects.Client()), fetch.WithMaxRetries(0)) - proxy.Fetcher = fetcher - s.proxy, s.store = proxy, store - h := NewGitHandler(proxy, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ - "up": {URL: s.git.URL}, - }}).WithBaseURL("http://proxy.test/") - routes := h.Routes() - // As the server marks client requests. - s.client = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := StreamCacheMiss(r.Context()) - if detached { - ctx = DetachUpstreamFetch(ctx) - } - routes.ServeHTTP(w, r.WithContext(ctx)) - })) - s.path = fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", s.oid, len(streamMissContent)) - t.Cleanup(func() { - s.releaseUpstream() - s.client.Close() - s.git.Close() - s.objects.Close() - _ = fetcher.Close() - }) - return s -} - -func (s *streamMissSetup) releaseUpstream() { - select { - case <-s.release: - default: - close(s.release) - } -} - -// get requests the object. The response headers must arrive in time even -// while the upstream holds back the second half. -func (s *streamMissSetup) get(t *testing.T) *http.Response { - t.Helper() - type result struct { - resp *http.Response - err error - } - got := make(chan result, 1) - go func() { - resp, err := s.client.Client().Get(s.client.URL + s.path) - got <- result{resp, err} - }() - select { - case r := <-got: - if r.err != nil { - t.Fatal(r.err) - } - return r.resp - case <-time.After(5 * time.Second): - // Once the upstream is released the response arrives; close it. - t.Cleanup(func() { - s.releaseUpstream() - if r := <-got; r.resp != nil { - _ = r.resp.Body.Close() - } - }) - t.Fatal("no response while the upstream was still sending") - return nil - } -} - -// cached waits for the stored copy, which is written after the client's -// stream ends. -func (s *streamMissSetup) cached(t *testing.T) bool { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for { - res, err := s.proxy.GetCachedArtifact(t.Context(), gitEcosystem, "up/acme/widget", s.oid, s.oid) - if err == nil && res != nil { - _ = res.Reader.Close() - return true - } - if time.Now().After(deadline) { - return false - } - time.Sleep(10 * time.Millisecond) - } -} - -// readFirst reads the first streamMissPrefix bytes of body, failing if they -// don't arrive in time. -func readFirst(t *testing.T, body io.Reader) string { - t.Helper() - got := make(chan string, 1) - go func() { - buf := make([]byte, streamMissPrefix) - _, _ = io.ReadFull(body, buf) - got <- string(buf) - }() - select { - case s := <-got: - return s - case <-time.After(5 * time.Second): - t.Fatal("no bytes reached the client while the upstream was still sending") - return "" - } -} - -func TestCacheMissStreamsBeforeUpstreamFinishes(t *testing.T) { - s := newStreamMiss(t, streamMissContent) - resp := s.get(t) - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK || resp.ContentLength != -1 { - t.Fatalf("status %d, Content-Length %d; want 200, chunked", resp.StatusCode, resp.ContentLength) - } - if first := readFirst(t, resp.Body); first != streamMissContent[:streamMissPrefix] { - t.Fatal("first bytes differ from the artifact's") - } - - s.releaseUpstream() - rest, err := io.ReadAll(resp.Body) - if err != nil || string(rest) != streamMissContent[streamMissPrefix:] { - t.Fatalf("rest: %d bytes, %v", len(rest), err) - } - if !s.cached(t) { - t.Fatal("streamed artifact was not cached") - } - - // Served from the cache now, with its size. - s.objects.Close() - again := s.get(t) - body, _ := io.ReadAll(again.Body) - _ = again.Body.Close() - if again.StatusCode != http.StatusOK || string(body) != streamMissContent || again.ContentLength != int64(len(streamMissContent)) { - t.Errorf("cached GET: status %d, length %d, %d bytes", again.StatusCode, again.ContentLength, len(body)) - } - if s.downloads.Load() != 1 { - t.Errorf("downloads = %d, want 1", s.downloads.Load()) - } -} - -func TestCacheMissStreamAbortedOnMismatch(t *testing.T) { - tampered := "X" + streamMissContent[1:] - s := newStreamMiss(t, tampered) - s.releaseUpstream() - resp := s.get(t) - body, err := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if err == nil { - t.Fatalf("read %d bytes without error; want the response aborted", len(body)) - } - if string(body) == tampered { - t.Error("client got every byte of an artifact that doesn't match its object ID") - } - // Nothing to wait for: storeArtifact has refused it once the stream ends. - time.Sleep(100 * time.Millisecond) - if res, _ := s.proxy.GetCachedArtifact(t.Context(), gitEcosystem, "up/acme/widget", s.oid, s.oid); res != nil { - _ = res.Reader.Close() - t.Error("mismatching artifact was cached") - } -} - -func TestCacheMissStreamFinishesWhenStorageFails(t *testing.T) { - s := newStreamMiss(t, streamMissContent) - s.store.mu.Lock() - s.store.storeErr = errors.New("no space left on device") - s.store.mu.Unlock() - before := testutil.ToFloat64(metrics.ArtifactsServedUncached.WithLabelValues(gitEcosystem)) - s.releaseUpstream() - - resp := s.get(t) - body, err := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if err != nil || resp.StatusCode != http.StatusOK || string(body) != streamMissContent { - t.Fatalf("status %d, %d bytes, %v; want the whole artifact", resp.StatusCode, len(body), err) - } - deadline := time.Now().Add(5 * time.Second) - for testutil.ToFloat64(metrics.ArtifactsServedUncached.WithLabelValues(gitEcosystem)) == before && time.Now().Before(deadline) { - time.Sleep(10 * time.Millisecond) - } - if got := testutil.ToFloat64(metrics.ArtifactsServedUncached.WithLabelValues(gitEcosystem)) - before; got != 1 { - t.Errorf("served-uncached count went up by %v, want 1", got) - } - if !strings.Contains(s.logs.String(), "level=ERROR") || !strings.Contains(s.logs.String(), "not cached") { - t.Errorf("storage failure not logged at error level:\n%s", s.logs.String()) - } -} - -func TestCacheMissStoredAfterClientLeaves(t *testing.T) { - s := newStreamMiss(t, streamMissContent) - resp := s.get(t) - readFirst(t, resp.Body) - _ = resp.Body.Close() - s.client.CloseClientConnections() - - s.releaseUpstream() - if !s.cached(t) { - t.Fatal("artifact not cached after the client that started the fetch left") - } -} - -func TestCacheMissWaiterGetsStoredArtifact(t *testing.T) { - s := newStreamMiss(t, streamMissContent) - first := s.get(t) - defer func() { _ = first.Body.Close() }() - readFirst(t, first.Body) - - second := make(chan string, 1) - go func() { - resp, err := s.client.Client().Get(s.client.URL + s.path) - if err != nil { - second <- err.Error() - return - } - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - second <- fmt.Sprintf("%d %d %s", resp.StatusCode, resp.ContentLength, body) - }() - select { - case got := <-second: - t.Fatalf("second request answered before the artifact was stored: %s", got) - case <-time.After(200 * time.Millisecond): - } - - s.releaseUpstream() - if _, err := io.ReadAll(first.Body); err != nil { - t.Fatal(err) - } - want := fmt.Sprintf("200 %d %s", len(streamMissContent), streamMissContent) - if got := <-second; got != want { - t.Errorf("second request = %.60q..., want the stored artifact with its length", got) - } - if s.downloads.Load() != 1 { - t.Errorf("downloads = %d, want 1", s.downloads.Load()) - } -} - -func newStreamMissScanGroup(t *testing.T, mode string) *scanner.Group { - t.Helper() - g, err := scanner.NewGroup(config.ScanningConfig{ - Enabled: true, - Timeout: "15s", - SigningKey: "test-signing-key", - Scanners: []config.ScannerConfig{ - {Name: "test-scanner", URL: newTestScanServer(t, true, "").URL, Mode: mode}, - }, - }, slog.New(slog.NewTextHandler(io.Discard, nil))) - if err != nil { - t.Fatal(err) - } - return g -} - -// A block-mode scanner must see the whole artifact before any of it is -// served, so a miss isn't streamed. -func TestCacheMissNotStreamedWithBlockingScanner(t *testing.T) { - s := newStreamMiss(t, streamMissContent) - s.proxy.ScanSigningKey = []byte("test-signing-key") - s.proxy.Scanners = newStreamMissScanGroup(t, "block") - - got := make(chan *http.Response, 1) - go func() { - resp, err := s.client.Client().Get(s.client.URL + s.path) - if err != nil { - t.Error(err) - } - got <- resp - }() - select { - case resp := <-got: - _ = resp.Body.Close() - t.Fatal("response started before the upstream finished, with a block-mode scanner") - case <-time.After(300 * time.Millisecond): - } - s.releaseUpstream() - resp := <-got - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if resp.StatusCode != http.StatusOK || string(body) != streamMissContent || resp.ContentLength != int64(len(streamMissContent)) { - t.Errorf("status %d, length %d, %d bytes; want the scanned, stored artifact", resp.StatusCode, resp.ContentLength, len(body)) - } -} - -// A monitor-mode scanner doesn't gate the artifact, so the miss is streamed -// and the scan runs after the client's stream ends. -func TestCacheMissStreamedWithMonitoringScanner(t *testing.T) { - s := newStreamMiss(t, streamMissContent) - s.proxy.ScanSigningKey = []byte("test-signing-key") - s.proxy.Scanners = newStreamMissScanGroup(t, "monitor") - - resp := s.get(t) - defer func() { _ = resp.Body.Close() }() - readFirst(t, resp.Body) - s.releaseUpstream() - if _, err := io.ReadAll(resp.Body); err != nil { - t.Fatal(err) - } - if !s.cached(t) { - t.Error("artifact not cached") - } -} - -// slowStorage stores like mockStorage, but takes a while to finish once it -// has read the last byte, as an upload to S3 does. -type slowStorage struct { - *mockStorage - delay time.Duration -} - -func (s *slowStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { - data, err := io.ReadAll(r) - if err != nil { - return 0, "", err - } - select { - case <-time.After(s.delay): - case <-ctx.Done(): - return 0, "", ctx.Err() - } - return s.mockStorage.Store(ctx, path, bytes.NewReader(data)) -} - -// Without a detached fetch, storing still finishes after the client's -// request has ended with the last byte. -func TestCacheMissStoredAfterResponseWithoutDetach(t *testing.T) { - s := newStreamMissMarked(t, streamMissContent, false) - s.proxy.Storage = &slowStorage{mockStorage: s.store, delay: 300 * time.Millisecond} - s.releaseUpstream() - resp := s.get(t) - body, err := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if err != nil || string(body) != streamMissContent { - t.Fatalf("%d bytes, %v", len(body), err) - } - if !s.cached(t) { - t.Fatal("artifact not cached once the response ended") - } -} - -// lowerFeedLimit makes a client fall behind after n bytes. -func lowerFeedLimit(t *testing.T, n int) { - t.Helper() - old := clientFeedLimit - clientFeedLimit = n - t.Cleanup(func() { clientFeedLimit = old }) -} - -// TestCacheMissSlowClientDoesNotHoldBackFetch has the caller that starts a -// streamed miss read nothing until a second caller, waiting on the same -// fetch, has the stored artifact. The slow client then gets every byte, the -// ones past the limit from the stored copy. -func TestCacheMissSlowClientDoesNotHoldBackFetch(t *testing.T) { - for _, seekable := range []bool{false, true} { - t.Run(fmt.Sprintf("seekable=%v", seekable), func(t *testing.T) { testSlowClient(t, seekable) }) - } -} - -func testSlowClient(t *testing.T, seekable bool) { - t.Helper() - lowerFeedLimit(t, 64<<10) - proxy, _, store, _ := setupTestProxy(t) - store.seekable = seekable - fetcher := newGatedFetcher(streamMissContent) - proxy.Fetcher = fetcher - get := func(ctx context.Context) (*CacheResult, error) { - return proxy.GetOrFetchArtifactFromURL(ctx, "npm", "big", "1.0.0", "big-1.0.0.tgz", "https://example.com/big.tgz") - } - - slow := make(chan *CacheResult, 1) - go func() { - res, err := get(StreamCacheMiss(context.Background())) - if err != nil { - t.Errorf("slow client: %v", err) - } - slow <- res - }() - <-fetcher.entered - waiter := make(chan *CacheResult, 1) - go func() { - res, err := get(context.Background()) - if err != nil { - t.Errorf("waiter: %v", err) - } - waiter <- res - }() - waitForWaiters(t, proxy, 1) - close(fetcher.release) - - lead := <-slow - if lead == nil { - t.FailNow() - } - defer func() { _ = lead.Reader.Close() }() - if lead.Artifact.Size != -1 { - t.Fatalf("slow client got size %d, want a stream", lead.Artifact.Size) - } - select { - case res := <-waiter: - if res == nil { - t.FailNow() - } - body, err := io.ReadAll(res.Reader) - _ = res.Reader.Close() - if err != nil || string(body) != streamMissContent { - t.Fatalf("waiter: %d bytes, %v", len(body), err) - } - case <-time.After(5 * time.Second): - t.Fatal("waiter held back by a client that reads nothing") - } - - body, err := io.ReadAll(lead.Reader) - if err != nil || string(body) != streamMissContent { - t.Fatalf("slow client: %d bytes, %v; want the whole artifact", len(body), err) - } - if fetcher.calls.Load() != 1 { - t.Errorf("upstream fetches = %d, want 1", fetcher.calls.Load()) - } -} - -// TestCacheMissSlowClientFailsWithoutStoredCopy: a client that fell behind -// has nothing to continue from when storing fails, so its response fails -// rather than ending short. -func TestCacheMissSlowClientFailsWithoutStoredCopy(t *testing.T) { - lowerFeedLimit(t, 64<<10) - proxy, _, store, _ := setupTestProxy(t) - store.storeErr = errors.New("no space left on device") - proxy.Fetcher = &countingFetcher{content: streamMissContent} - - res, err := proxy.GetOrFetchArtifactFromURL(StreamCacheMiss(context.Background()), "npm", "big", "1.0.0", "big-1.0.0.tgz", "https://example.com/big.tgz") - if err != nil { - t.Fatal(err) - } - defer func() { _ = res.Reader.Close() }() - - // Leave the stream unread until the fetch ends so the feed must overflow. - proxy.fetchMu.Lock() - var done <-chan struct{} - for _, f := range proxy.inFlight { - done = f.done - } - proxy.fetchMu.Unlock() - if done != nil { - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("fetch did not finish while the client was idle") - } - } - - body, err := io.ReadAll(res.Reader) - if !errors.Is(err, store.storeErr) { - t.Fatalf("read %d bytes with error %v; want %v", len(body), err, store.storeErr) - } -} - -func TestMissFeedCloseWhileWaiting(t *testing.T) { - m := newMissFeed(4) - done := make(chan error, 1) - go func() { - _, err := m.Read(make([]byte, 8)) - done <- err - }() - time.Sleep(10 * time.Millisecond) - _ = m.Close() - select { - case err := <-done: - if !errors.Is(err, io.ErrClosedPipe) { - t.Errorf("Read after Close = %v", err) - } - case <-time.After(time.Second): - t.Fatal("Read still blocked after Close") - } - if m.push([]byte("ab")) { - t.Error("push to a closed feed reported the client fed") - } - m.resolve(io.NopCloser(strings.NewReader("x")), nil) -} diff --git a/internal/handler/swift.go b/internal/handler/swift.go deleted file mode 100644 index 37f666e..0000000 --- a/internal/handler/swift.go +++ /dev/null @@ -1,714 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "strconv" - "strings" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/proxy/internal/retention" - "github.com/git-pkgs/purl" -) - -const ( - swiftAcceptJSON = "application/vnd.swift.registry.v1+json" - swiftAcceptManifest = "application/vnd.swift.registry.v1+swift" - swiftAcceptArchive = "application/vnd.swift.registry.v1+zip" - swiftContentVersion = "1" - swiftMaxScopeLength = 39 - swiftMaxNameLength = 100 -) - -// Source archives are stored under the registry identity, -// pkg:generic/swift-registry/{scope}.{name}, lowercased. Package rules write -// the scope as the namespace, pkg:swift/apple/swift-nio, or use the stored -// form. A source repository such as pkg:swift/github.com/apple/swift-nio -// names no registry identity; it is resolved through the upstream's -// /identifiers before it applies. -func init() { - retention.Default.Register(retention.Spec{ - Key: "swift", - CanonicalPackage: swiftRetentionKey, - SourcePackage: func(p *purl.PURL) bool { return SwiftSourcePURL(p) != "" }, - }) -} - -// SwiftSourceURL returns the repository URL of a pkg:swift name whose -// namespace starts with a host, as in github.com/apple/swift-nio, or "" for -// a name whose namespace is a registry scope. -func SwiftSourceURL(name string) string { - if host, _, ok := strings.Cut(name, "/"); ok && strings.Contains(host, ".") { - return "https://" + name - } - return "" -} - -// SwiftSourcePURL returns the repository URL of a pkg:swift PURL that names -// a package by its source repository, as SBOM tools and GHSA do, or "". -func SwiftSourcePURL(p *purl.PURL) string { - if p == nil || p.Type != "swift" || p.Namespace == "" { - return "" - } - return SwiftSourceURL(p.Namespace + "/" + p.Name) -} - -// ResolveSourcePURL returns p with a source repository namespace replaced by -// each registry scope and name the upstream lists for it, and p itself when -// it already names a registry identity. A policy for the repository applies -// to all of them, since a client may use any. -func (h *SwiftHandler) ResolveSourcePURL(ctx context.Context, p *purl.PURL) ([]*purl.PURL, error) { - source := SwiftSourcePURL(p) - if source == "" { - return []*purl.PURL{p}, nil - } - ids, err := h.ResolveIdentities(ctx, source) - if err != nil { - return nil, err - } - resolved := make([]*purl.PURL, 0, len(ids)) - for _, id := range ids { - resolved = append(resolved, purl.New("swift", id.Scope, id.Name, p.Version, nil)) - } - return resolved, nil -} - -// swiftRetentionKey maps a package rule to the stored package PURL, refusing -// keys that name no valid registry scope and package. MakeCacheStrings -// lowercases the identity the way the handler stores it. -func swiftRetentionKey(p *purl.PURL) (string, bool) { - if p == nil { - return "", false - } - var scope, name string - switch { - case p.Type == "swift": - scope, name = p.Namespace, p.Name - case p.Type == "generic" && strings.EqualFold(p.Namespace, "swift-registry"): - var found bool - if scope, name, found = strings.Cut(p.Name, "."); !found { - return "", false - } - default: - return "", false - } - if !validSwiftScope(scope) || !validSwiftPackageName(name) { - return "", false - } - packagePURL, _ := packageurl.MakeCacheStrings("swift", scope+"/"+name, "") - return packagePURL, packagePURL != "" -} - -// SwiftHandler handles the read-only Swift Package Registry v1 protocol. -type SwiftHandler struct { - proxy *Proxy - upstreamURL string - proxyURL string -} - -// NewSwiftHandler creates a Swift Package Registry protocol handler. -func NewSwiftHandler(proxy *Proxy, proxyURL, upstreamURL string) *SwiftHandler { - if strings.TrimSpace(upstreamURL) == "" { - upstreamURL = config.DefaultSwiftUpstream - } - - return &SwiftHandler{ - proxy: proxy, - upstreamURL: strings.TrimSuffix(upstreamURL, "/"), - proxyURL: strings.TrimSuffix(proxyURL, "/"), - } -} - -// Routes returns the HTTP handler for Swift registry requests. -func (h *SwiftHandler) Routes() http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("GET /identifiers", h.handleIdentifiers) - mux.HandleFunc("GET /{scope}/{name}/{version}/Package.swift", h.handleManifest) - mux.HandleFunc("GET /{scope}/{name}/{version}", h.handleRelease) - mux.HandleFunc("PUT /{scope}/{name}/{version}", h.handlePublishingUnsupported) - mux.HandleFunc("GET /{scope}/{name}", h.handlePackageReleases) - return mux -} - -func (h *SwiftHandler) handlePackageReleases(w http.ResponseWriter, r *http.Request) { - scope := r.PathValue("scope") - name := strings.TrimSuffix(r.PathValue("name"), ".json") - if !validSwiftScope(scope) || !validSwiftPackageName(name) { - writeSwiftProblem(w, http.StatusBadRequest, "invalid package identifier") - return - } - scope, name = canonicalSwiftPackage(scope, name) - - upstreamURL := h.buildUpstreamURL(scope, name, "", "", r.URL.RawQuery) - doc, err := h.fetchDocument( - r.Context(), swiftReleasesCacheKey(scope, name, r.URL.RawQuery), upstreamURL, requestAccept(r, swiftAcceptJSON), - ) - if err != nil { - h.writeMetadataError(w, err) - return - } - - rewritten, err := h.rewriteReleaseURLs(scope, name, doc.body) - if err != nil { - h.proxy.Logger.Warn("failed to rewrite Swift release URLs", "error", err) - rewritten = doc.body - } - if doc.link != "" { - w.Header().Set("Link", h.rewriteLinkHeader(doc.link, upstreamURL)) - } - writeSwiftMetadata(w, r, rewritten, doc.contentType) -} - -func (h *SwiftHandler) handleRelease(w http.ResponseWriter, r *http.Request) { - scope := r.PathValue("scope") - name := r.PathValue("name") - version := r.PathValue("version") - if strings.HasSuffix(version, ".zip") { - h.handleSourceArchive(w, r, scope, name, strings.TrimSuffix(version, ".zip")) - return - } - - version = strings.TrimSuffix(version, ".json") - if !validSwiftPackageReference(scope, name, version) { - writeSwiftProblem(w, http.StatusBadRequest, "invalid package release") - return - } - scope, name = canonicalSwiftPackage(scope, name) - - upstreamURL := h.buildUpstreamURL(scope, name, version, "", r.URL.RawQuery) - body, contentType, err := h.proxy.FetchOrCacheMetadata( - r.Context(), "swift", swiftReleaseCacheKey(scope, name, version), upstreamURL, requestAccept(r, swiftAcceptJSON), - ) - if err != nil { - h.writeMetadataError(w, err) - return - } - writeSwiftMetadata(w, r, body, contentType) -} - -func (h *SwiftHandler) handleManifest(w http.ResponseWriter, r *http.Request) { - scope := r.PathValue("scope") - name := r.PathValue("name") - version := r.PathValue("version") - if !validSwiftPackageReference(scope, name, version) { - writeSwiftProblem(w, http.StatusBadRequest, "invalid package release") - return - } - scope, name = canonicalSwiftPackage(scope, name) - - upstreamURL := h.buildUpstreamURL(scope, name, version, "Package.swift", r.URL.RawQuery) - doc, err := h.fetchDocument( - r.Context(), swiftManifestCacheKey(scope, name, version, r.URL.RawQuery), upstreamURL, requestAccept(r, swiftAcceptManifest), - ) - if err != nil { - h.writeMetadataError(w, err) - return - } - if doc.link != "" { - w.Header().Set("Link", h.rewriteLinkHeader(doc.link, upstreamURL)) - } - writeSwiftMetadata(w, r, doc.body, doc.contentType) -} - -func (h *SwiftHandler) handleIdentifiers(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("url") == "" { - writeSwiftProblem(w, http.StatusBadRequest, "url query parameter is required") - return - } - - upstreamURL := h.upstreamURL + "/identifiers?" + r.URL.RawQuery - cacheKey := swiftMetadataCacheKey("identifiers", r.URL.RawQuery) - body, contentType, err := h.proxy.FetchOrCacheMetadata( - r.Context(), "swift", cacheKey, upstreamURL, requestAccept(r, swiftAcceptJSON), - ) - if err != nil { - h.writeMetadataError(w, err) - return - } - writeSwiftMetadata(w, r, body, contentType) -} - -func (h *SwiftHandler) handlePublishingUnsupported(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Allow", "GET, HEAD") - writeSwiftProblem(w, http.StatusMethodNotAllowed, "publishing isn't supported") -} - -func (h *SwiftHandler) handleSourceArchive(w http.ResponseWriter, r *http.Request, scope, name, version string) { - if !validSwiftPackageReference(scope, name, version) { - writeSwiftProblem(w, http.StatusBadRequest, "invalid package release") - return - } - scope, name = canonicalSwiftPackage(scope, name) - - key, err := swiftSourceArchiveKey(scope, name, version) - if err != nil { - h.writeArtifactError(w, err) - return - } - filename, packagePURL, versionPURL := key.filename, key.packagePURL, key.versionPURL - upstreamURL := h.buildUpstreamURL(scope, name, version+".zip", "", r.URL.RawQuery) - archiveInfo, infoErr := h.fetchArchiveInfo(r.Context(), scope, name, version) - if infoErr != nil { - h.writeArtifactError(w, fmt.Errorf("fetching release metadata: %w", infoErr)) - return - } - - if r.Method == http.MethodHead { - h.handleSourceArchiveHead(w, r, name, version, filename, packagePURL, versionPURL, upstreamURL, archiveInfo) - return - } - - headers := make(http.Header) - headers.Set("Accept", requestAccept(r, swiftAcceptArchive)) - result, err := h.fetchSourceArchive(r.Context(), key, version, upstreamURL, headers, archiveInfo) - if err != nil { - h.writeArtifactError(w, err) - return - } - - result.Artifact.MediaType = "application/zip" - setSwiftArchiveHeaders(w.Header(), name, version, result.Artifact.Digest.Encoded(), archiveInfo) - ServeArtifactRequest(w, r, result) -} - -// swiftArchiveKey is the cache key a release's source archive is stored -// under. -type swiftArchiveKey struct { - packageName, filename, packagePURL, versionPURL string -} - -// swiftSourceArchiveKey returns the cache key of the source archive of the -// canonical (lowercased) scope and name at version. -func swiftSourceArchiveKey(scope, name, version string) (swiftArchiveKey, error) { - key := swiftArchiveKey{packageName: scope + "/" + name, filename: fmt.Sprintf("%s-%s.zip", name, version)} - key.packagePURL, key.versionPURL = packageurl.MakeCacheStrings("swift", key.packageName, version) - if key.packagePURL == "" || key.versionPURL == "" { - return swiftArchiveKey{}, fmt.Errorf("%w: swift %q", errUnsupportedPackageIdentity, key.packageName) - } - return key, nil -} - -// fetchSourceArchive returns the source archive under key from the cache, or -// fetches it from upstreamURL, verified against the checksum in info. -func (h *SwiftHandler) fetchSourceArchive(ctx context.Context, key swiftArchiveKey, version, upstreamURL string, headers http.Header, info swiftArchiveInfo) (*CacheResult, error) { - return h.proxy.getOrFetchArtifactFromURLWithCachePURLs( - ctx, "swift", key.packageName, version, key.filename, key.packagePURL, key.versionPURL, - upstreamURL, headers, info.checksum, "", - ) -} - -func (h *SwiftHandler) handleSourceArchiveHead( - w http.ResponseWriter, - r *http.Request, - name, version, filename, packagePURL, versionPURL, upstreamURL string, - archiveInfo swiftArchiveInfo, -) { - result, err := h.proxy.getCachedArtifactWithUpstreamHash( - r.Context(), packagePURL, versionPURL, filename, archiveInfo.checksum, - ) - if err != nil { - h.writeArtifactError(w, err) - return - } - if result != nil { - result.Artifact.MediaType = "application/zip" - setSwiftArchiveHeaders(w.Header(), name, version, result.Artifact.Digest.Encoded(), archiveInfo) - ServeArtifactRequest(w, r, result) - return - } - - size, err := h.probeSourceArchive(r.Context(), upstreamURL, requestAccept(r, swiftAcceptArchive)) - if err != nil { - h.writeArtifactError(w, err) - return - } - setSwiftArchiveHeaders(w.Header(), name, version, "", archiveInfo) - w.Header().Set(headerContentType, "application/zip") - if size >= 0 { - w.Header().Set(headerContentLength, strconv.FormatInt(size, 10)) - } - w.WriteHeader(http.StatusOK) -} - -func (h *SwiftHandler) probeSourceArchive(ctx context.Context, upstreamURL, accept string) (int64, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) - if err != nil { - return 0, fmt.Errorf("creating upstream archive request: %w", err) - } - req.Header.Set("Accept", accept) - req.Header.Set("Range", "bytes=0-0") - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return 0, fmt.Errorf("requesting upstream archive: %w", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode == http.StatusNotFound { - return 0, ErrUpstreamNotFound - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - return 0, fmt.Errorf("upstream archive returned %d", resp.StatusCode) - } - - if resp.StatusCode == http.StatusPartialContent { - _, total, found := strings.Cut(resp.Header.Get("Content-Range"), "/") - if !found || total == "*" { - return -1, nil - } - if parsed, parseErr := strconv.ParseInt(total, 10, 64); parseErr == nil { - return parsed, nil - } - return -1, nil - } - - size := int64(-1) - if contentLength := resp.Header.Get(headerContentLength); contentLength != "" { - if parsed, parseErr := strconv.ParseInt(contentLength, 10, 64); parseErr == nil { - size = parsed - } - } - return size, nil -} - -func (h *SwiftHandler) fetchMetadataWithHeaders( - ctx context.Context, - upstreamURL, accept string, -) ([]byte, string, http.Header, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) - if err != nil { - return nil, "", nil, fmt.Errorf("creating upstream metadata request: %w", err) - } - req.Header.Set("Accept", accept) - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return nil, "", nil, fmt.Errorf("requesting upstream metadata: %w", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode == http.StatusNotFound { - return nil, "", nil, ErrUpstreamNotFound - } - if resp.StatusCode != http.StatusOK { - return nil, "", nil, fmt.Errorf("upstream metadata returned %d", resp.StatusCode) - } - - body, err := h.proxy.ReadMetadata(resp.Body) - if err != nil { - return nil, "", nil, fmt.Errorf("reading upstream metadata: %w", err) - } - contentType := resp.Header.Get(headerContentType) - if contentType == "" { - contentType = contentTypeJSON - } - return body, contentType, resp.Header.Clone(), nil -} - -type swiftReleaseMetadata struct { - Resources []struct { - Name string `json:"name"` - Type string `json:"type"` - Checksum string `json:"checksum"` - Signing *struct { - Signature string `json:"signatureBase64Encoded"` - Format string `json:"signatureFormat"` - } `json:"signing"` - } `json:"resources"` -} - -type swiftArchiveInfo struct { - checksum string - signature string - signatureFormat string -} - -func (h *SwiftHandler) fetchArchiveInfo(ctx context.Context, scope, name, version string) (swiftArchiveInfo, error) { - upstreamURL := h.buildUpstreamURL(scope, name, version, "", "") - body, _, err := h.proxy.FetchOrCacheMetadata( - ctx, "swift", swiftReleaseCacheKey(scope, name, version), upstreamURL, swiftAcceptJSON, - ) - if err != nil { - return swiftArchiveInfo{}, err - } - - var metadata swiftReleaseMetadata - if err := json.Unmarshal(body, &metadata); err != nil { - return swiftArchiveInfo{}, fmt.Errorf("parsing release metadata: %w", err) - } - for _, resource := range metadata.Resources { - if resource.Name != "source-archive" || resource.Type != "application/zip" { - continue - } - checksum, err := normalizeSwiftChecksum(resource.Checksum) - if err != nil { - return swiftArchiveInfo{}, err - } - info := swiftArchiveInfo{checksum: checksum} - if resource.Signing != nil { - if resource.Signing.Signature == "" || resource.Signing.Format == "" { - return swiftArchiveInfo{}, errors.New("source archive signing metadata is incomplete") - } - info.signature = resource.Signing.Signature - info.signatureFormat = resource.Signing.Format - } - return info, nil - } - - return swiftArchiveInfo{}, errors.New("source archive is missing from release metadata") -} - -func normalizeSwiftChecksum(checksum string) (string, error) { - digest, err := hex.DecodeString(checksum) - if err != nil || len(digest) != sha256.Size { - return "", errors.New("source archive checksum is not a SHA-256 digest") - } - return hex.EncodeToString(digest), nil -} - -func setSwiftArchiveHeaders(header http.Header, name, version, contentHash string, info swiftArchiveInfo) { - header.Set("Cache-Control", "public, immutable") - header.Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s-%s.zip"`, name, version)) - header.Set("Content-Version", swiftContentVersion) - - checksum := info.checksum - if checksum == "" { - checksum = contentHash - } - if digest := swiftDigestHeader(checksum); digest != "" { - header.Set("Digest", digest) - } - if info.signature != "" && info.signatureFormat != "" { - header.Set("X-Swift-Package-Signature", info.signature) - header.Set("X-Swift-Package-Signature-Format", info.signatureFormat) - } -} - -func swiftDigestHeader(checksum string) string { - digest, err := hex.DecodeString(checksum) - if err != nil || len(digest) != sha256.Size { - return "" - } - return "sha-256=" + base64.StdEncoding.EncodeToString(digest) -} - -func (h *SwiftHandler) rewriteLinkHeader(value, upstreamRequestURL string) string { - var result strings.Builder - for len(value) > 0 { - start := strings.IndexByte(value, '<') - if start < 0 { - result.WriteString(value) - break - } - endOffset := strings.IndexByte(value[start+1:], '>') - if endOffset < 0 { - result.WriteString(value) - break - } - end := start + 1 + endOffset - result.WriteString(value[:start+1]) - result.WriteString(h.rewriteRegistryURL(value[start+1:end], upstreamRequestURL)) - result.WriteByte('>') - value = value[end+1:] - } - return result.String() -} - -func (h *SwiftHandler) rewriteRegistryURL(rawURL, upstreamRequestURL string) string { - base, err := url.Parse(h.upstreamURL) - if err != nil { - return rawURL - } - requestURL, err := url.Parse(upstreamRequestURL) - if err != nil { - return rawURL - } - reference, err := url.Parse(rawURL) - if err != nil { - return rawURL - } - absolute := requestURL.ResolveReference(reference) - if !strings.EqualFold(absolute.Scheme, base.Scheme) || !strings.EqualFold(absolute.Host, base.Host) { - return rawURL - } - - basePath := strings.TrimSuffix(base.EscapedPath(), "/") - absolutePath := absolute.EscapedPath() - if absolutePath != basePath && !strings.HasPrefix(absolutePath, basePath+"/") { - return rawURL - } - suffix := strings.TrimPrefix(absolutePath, basePath) - rewritten := h.proxyURL + "/swift" + suffix - if absolute.RawQuery != "" { - rewritten += "?" + absolute.RawQuery - } - if absolute.Fragment != "" { - rewritten += "#" + absolute.Fragment - } - return rewritten -} - -func (h *SwiftHandler) rewriteReleaseURLs(scope, name string, body []byte) ([]byte, error) { - var metadata map[string]any - if err := json.Unmarshal(body, &metadata); err != nil { - return nil, err - } - releases, ok := metadata["releases"].(map[string]any) - if !ok { - return body, nil - } - - for version, value := range releases { - release, ok := value.(map[string]any) - if !ok { - continue - } - if _, hasURL := release["url"]; !hasURL { - continue - } - release["url"] = fmt.Sprintf( - "%s/swift/%s/%s/%s", - h.proxyURL, - url.PathEscape(scope), - url.PathEscape(name), - url.PathEscape(version), - ) - } - return json.Marshal(metadata) -} - -func (h *SwiftHandler) buildUpstreamURL(scope, name, version, resource, rawQuery string) string { - parts := []string{h.upstreamURL, url.PathEscape(scope), url.PathEscape(name)} - if version != "" { - parts = append(parts, url.PathEscape(version)) - } - if resource != "" { - parts = append(parts, resource) - } - result := strings.Join(parts, "/") - if rawQuery != "" { - result += "?" + rawQuery - } - return result -} - -func swiftMetadataCacheKey(parts ...string) string { - joined := strings.Join(parts, "\x00") - digest := sha256.Sum256([]byte(joined)) - return hex.EncodeToString(digest[:]) -} - -func swiftReleaseCacheKey(scope, name, version string) string { - return swiftMetadataCacheKey("release", scope, name, version) -} - -func requestAccept(r *http.Request, fallback string) string { - if accept := r.Header.Get("Accept"); accept != "" { - return accept - } - return fallback -} - -func writeSwiftMetadata(w http.ResponseWriter, r *http.Request, body []byte, contentType string) { - if contentType == "" { - contentType = "application/json" - } - digest := sha256.Sum256(body) - etag := fmt.Sprintf(`"%x"`, digest) - w.Header().Set(headerContentType, contentType) - w.Header().Set("Content-Version", swiftContentVersion) - w.Header().Set(headerETag, etag) - if ifNoneMatchHits(r.Header.Get("If-None-Match"), etag) { - w.WriteHeader(http.StatusNotModified) - return - } - w.Header().Set(headerContentLength, strconv.Itoa(len(body))) - w.WriteHeader(http.StatusOK) - if r.Method != http.MethodHead { - _, _ = w.Write(body) - } -} - -func (h *SwiftHandler) writeMetadataError(w http.ResponseWriter, err error) { - if errors.Is(err, ErrUpstreamNotFound) { - writeSwiftProblem(w, http.StatusNotFound, "not found") - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("Swift metadata request failed", "error", err) - writeSwiftProblem(w, http.StatusBadGateway, "upstream request failed") -} - -func (h *SwiftHandler) writeArtifactError(w http.ResponseWriter, err error) { - if errors.Is(err, ErrUpstreamNotFound) { - writeSwiftProblem(w, http.StatusNotFound, "release not found") - return - } - if errors.Is(err, ErrArtifactBlocked) || errors.Is(err, ErrVersionDenied) { - writeSwiftProblem(w, http.StatusForbidden, err.Error()) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("Swift archive request failed", "error", err) - writeSwiftProblem(w, http.StatusBadGateway, "failed to fetch package") -} - -func writeSwiftProblem(w http.ResponseWriter, status int, detail string) { - w.Header().Set(headerContentType, "application/problem+json") - w.Header().Set("Content-Version", swiftContentVersion) - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(map[string]string{"detail": detail}) -} - -func validSwiftPackageReference(scope, name, version string) bool { - return validSwiftScope(scope) && validSwiftPackageName(name) && version != "" && version != "." && version != ".." && !strings.ContainsAny(version, "/\\") -} - -func canonicalSwiftPackage(scope, name string) (string, string) { - return strings.ToLower(scope), strings.ToLower(name) -} - -func validSwiftScope(scope string) bool { - return validSwiftIdentifier(scope, swiftMaxScopeLength, "-") -} - -func validSwiftPackageName(name string) bool { - return validSwiftIdentifier(name, swiftMaxNameLength, "-_") -} - -func validSwiftIdentifier(value string, maxLength int, separators string) bool { - if value == "" || len(value) > maxLength { - return false - } - previousSeparator := false - for i := 0; i < len(value); i++ { - character := value[i] - separator := strings.ContainsRune(separators, rune(character)) - if separator { - if i == 0 || i == len(value)-1 || previousSeparator { - return false - } - previousSeparator = true - continue - } - if (character < 'a' || character > 'z') && - (character < 'A' || character > 'Z') && - (character < '0' || character > '9') { - return false - } - previousSeparator = false - } - return true -} diff --git a/internal/handler/swift_cache.go b/internal/handler/swift_cache.go deleted file mode 100644 index d778179..0000000 --- a/internal/handler/swift_cache.go +++ /dev/null @@ -1,128 +0,0 @@ -package handler - -import ( - "bytes" - "context" - "database/sql" - "errors" - "net/url" - "strings" - "time" - - "github.com/git-pkgs/proxy/internal/database" -) - -// swiftDocument is a release list or manifest as the upstream sent it. -type swiftDocument struct { - body []byte - contentType string - // link is the upstream Link header, its values joined, with the - // upstream's URLs. - link string - fetchedAt time.Time -} - -// fetchDocument returns the document at upstreamURL, from the metadata cache -// under cacheKey while it is fresh or when the upstream fails, and otherwise -// from upstream, storing it. The upstream Link header is cached with it, -// since it carries the pagination of release lists and the version-specific -// manifests a client picks from. -func (h *SwiftHandler) fetchDocument(ctx context.Context, cacheKey, upstreamURL, accept string) (*swiftDocument, error) { - cached := h.loadDocument(ctx, cacheKey) - if cached != nil && h.proxy.MetadataTTL > 0 && time.Since(cached.fetchedAt) < h.proxy.MetadataTTL { - return cached, nil - } - body, contentType, headers, err := h.fetchMetadataWithHeaders(ctx, upstreamURL, accept) - if err != nil { - if cached == nil || errors.Is(err, ErrUpstreamNotFound) { - return nil, err - } - h.proxy.Logger.Warn("upstream Swift metadata fetch failed, serving cached copy", - "url", upstreamURL, "error", err) - return cached, nil - } - doc := &swiftDocument{ - body: body, - contentType: contentType, - link: strings.Join(headers.Values("Link"), ", "), - fetchedAt: time.Now(), - } - h.storeDocument(ctx, cacheKey, doc) - return doc, nil -} - -func (h *SwiftHandler) documentsCached() bool { - return h.proxy.CacheMetadata && h.proxy.DB != nil && h.proxy.Storage != nil -} - -// loadDocument returns the cached document under cacheKey, or nil. -func (h *SwiftHandler) loadDocument(ctx context.Context, cacheKey string) *swiftDocument { - if !h.documentsCached() { - return nil - } - entry := h.proxy.storedMetadataEntry("swift", cacheKey, true) - if entry == nil { - return nil - } - reader, err := h.proxy.openMetadataEntry(ctx, "swift", cacheKey, entry) - if err != nil { - return nil - } - defer func() { _ = reader.Close() }() - body, err := h.proxy.ReadMetadata(reader) - if err != nil { - return nil - } - h.proxy.recordMetadataRead(entry) - return &swiftDocument{ - body: body, - contentType: entry.ContentType.String, - link: entry.Link.String, - fetchedAt: entry.FetchedAt.Time, - } -} - -// storeDocument caches doc under cacheKey. A failure is logged; the -// document is still served. -func (h *SwiftHandler) storeDocument(ctx context.Context, cacheKey string, doc *swiftDocument) { - if !h.documentsCached() { - return - } - storagePath := metadataStoragePath("swift", cacheKey) - size, hash, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(doc.body)) - if err == nil { - err = h.proxy.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ - Ecosystem: "swift", - Name: cacheKey, - StoragePath: storagePath, - Link: sql.NullString{String: doc.link, Valid: doc.link != ""}, - ContentType: sql.NullString{String: doc.contentType, Valid: doc.contentType != ""}, - ContentDigest: sql.NullString{String: "sha256:" + hash, Valid: hash != ""}, - Size: sql.NullInt64{Int64: size, Valid: true}, - FetchedAt: sql.NullTime{Time: doc.fetchedAt, Valid: true}, - }) - } - if err != nil { - h.proxy.Logger.Warn("failed to cache Swift metadata", "key", cacheKey, "error", err) - } -} - -func swiftReleasesCacheKey(scope, name, rawQuery string) string { - return swiftMetadataCacheKey("releases", scope, name, rawQuery) -} - -// swiftManifestCacheKey keys a manifest by its swift-version as a tools -// version, so the 5.8 of a manifest's alternate link and the 5.8.0 a client -// requests for it share a key. -func swiftManifestCacheKey(scope, name, version, rawQuery string) string { - if query, err := url.ParseQuery(rawQuery); err == nil { - if tools := query.Get("swift-version"); tools != "" { - for strings.Count(tools, ".") > 0 && strings.HasSuffix(tools, ".0") { - tools = strings.TrimSuffix(tools, ".0") - } - query.Set("swift-version", tools) - } - rawQuery = query.Encode() - } - return swiftMetadataCacheKey("manifest", scope, name, version, rawQuery) -} diff --git a/internal/handler/swift_cache_test.go b/internal/handler/swift_cache_test.go deleted file mode 100644 index 2b194a5..0000000 --- a/internal/handler/swift_cache_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package handler - -import ( - "context" - "io" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" - "time" -) - -func TestSwiftReleaseListAndManifestServedFromCacheWhenUpstreamFails(t *testing.T) { - var down atomic.Bool - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if down.Load() { - http.Error(w, "down", http.StatusServiceUnavailable) - return - } - w.Header().Set("Link", `; rel="next"`) - if strings.HasSuffix(r.URL.Path, "/Package.swift") { - w.Header().Set("Content-Type", "text/x-swift") - _, _ = io.WriteString(w, "// swift-tools-version:5.9") - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"releases":{"1.0.0":{}}}`) - })) - defer upstream.Close() - - for _, cacheMetadata := range []bool{true, false} { - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = cacheMetadata - routes := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes() - get := func(path string) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - return w - } - down.Store(false) - for _, path := range []string{"/apple/example", "/apple/example/1.0.0/Package.swift"} { - if w := get(path); w.Code != http.StatusOK { - t.Fatalf("%s: status = %d with the upstream up", path, w.Code) - } - } - - down.Store(true) - for path, body := range map[string]string{"/apple/example": `"1.0.0"`, "/apple/example/1.0.0/Package.swift": "swift-tools-version:5.9"} { - w := get(path) - if !cacheMetadata { - if w.Code == http.StatusOK { - t.Errorf("%s: served without cache_metadata while the upstream is down", path) - } - continue - } - if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), body) { - t.Errorf("%s: status = %d, body %q; want the cached copy", path, w.Code, w.Body.String()) - } - if link := w.Header().Get("Link"); link != `; rel="next"` { - t.Errorf("%s: Link = %q, want the cached Link rewritten", path, link) - } - } - } -} - -// TestSwiftCorruptManifestRefetched checks that a cached manifest whose -// stored bytes no longer match the recorded digest is fetched again, even -// within metadata_ttl. -func TestSwiftCorruptManifestRefetched(t *testing.T) { - var hits atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - hits.Add(1) - w.Header().Set("Content-Type", "text/x-swift") - _, _ = io.WriteString(w, "// swift-tools-version:5.9") - })) - defer upstream.Close() - - proxy, db, store, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - routes := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes() - get := func() *httptest.ResponseRecorder { - w := httptest.NewRecorder() - routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.0.0/Package.swift", nil)) - return w - } - - get() - entry, err := db.GetMetadataCache("swift", swiftManifestCacheKey("apple", "example", "1.0.0", "")) - if err != nil || entry == nil { - t.Fatalf("manifest not cached: %v", err) - } - if !strings.HasPrefix(entry.ContentDigest.String, "sha256:") { - t.Fatalf("ContentDigest = %q, want a sha256 digest", entry.ContentDigest.String) - } - if w := get(); hits.Load() != 1 || w.Code != http.StatusOK { - t.Fatalf("fresh cached manifest: status %d, %d upstream fetches; want it served from the cache", w.Code, hits.Load()) - } - - if _, _, err := store.Store(context.Background(), entry.StoragePath, strings.NewReader("// tampered")); err != nil { - t.Fatal(err) - } - w := get() - if hits.Load() != 2 { - t.Errorf("upstream fetches = %d, want the corrupt manifest fetched again", hits.Load()) - } - if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), "swift-tools-version:5.9") { - t.Errorf("status = %d, body %q; want the upstream manifest", w.Code, w.Body.String()) - } -} - -func TestSwiftManifestCacheKeyIgnoresToolsVersionZeros(t *testing.T) { - key := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.8") - if got := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.8.0"); got != key { - t.Error("5.8.0 and 5.8 have different keys") - } - if got := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.9"); got == key { - t.Error("5.9 and 5.8 share a key") - } - if got := swiftManifestCacheKey("apple", "example", "1.0.0", ""); got == key { - t.Error("Package.swift and the 5.8 manifest share a key") - } -} diff --git a/internal/handler/swift_mirror.go b/internal/handler/swift_mirror.go deleted file mode 100644 index 5326c1d..0000000 --- a/internal/handler/swift_mirror.go +++ /dev/null @@ -1,187 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "strings" -) - -// MirrorRelease caches release version of scope.name so that a client can -// resolve and fetch the release without the upstream. -func (h *SwiftHandler) MirrorRelease(ctx context.Context, scope, name, version string) (*CacheResult, error) { - if !validSwiftPackageReference(scope, name, version) { - return nil, fmt.Errorf("invalid Swift registry release %s.%s@%s", scope, name, version) - } - scope, name = canonicalSwiftPackage(scope, name) - key, err := swiftSourceArchiveKey(scope, name, version) - if err != nil { - return nil, err - } - if h.documentsCached() { - if err := h.mirrorReleaseList(ctx, scope, name); err != nil { - return nil, fmt.Errorf("fetching release list: %w", err) - } - if err := h.mirrorManifests(ctx, scope, name, version); err != nil { - return nil, fmt.Errorf("fetching manifest: %w", err) - } - } - info, err := h.fetchArchiveInfo(ctx, scope, name, version) - if err != nil { - return nil, fmt.Errorf("fetching release metadata: %w", err) - } - headers := http.Header{"Accept": {swiftAcceptArchive}} - return h.fetchSourceArchive(ctx, key, version, h.buildUpstreamURL(scope, name, version+".zip", "", ""), headers, info) -} - -// maxSwiftReleasePages bounds the release list pages a mirror follows. -const maxSwiftReleasePages = 100 - -// mirrorReleaseList caches every page of the release list, under the keys -// the release list route reads for the page links it rewrites. -func (h *SwiftHandler) mirrorReleaseList(ctx context.Context, scope, name string) error { - query := "" - seen := map[string]bool{} - for page := 0; page < maxSwiftReleasePages && !seen[query]; page++ { - seen[query] = true - upstreamURL := h.buildUpstreamURL(scope, name, "", "", query) - doc, err := h.fetchDocument(ctx, swiftReleasesCacheKey(scope, name, query), upstreamURL, swiftAcceptJSON) - if err != nil { - return err - } - next := swiftLinkQueries(doc.link, "next", upstreamURL) - if len(next) == 0 { - return nil - } - query = next[0] - } - return nil -} - -// mirrorManifests caches Package.swift of a release and the version-specific -// manifests its Link header lists, under the keys the manifest route reads -// for the alternate links it rewrites. -func (h *SwiftHandler) mirrorManifests(ctx context.Context, scope, name, version string) error { - upstreamURL := h.buildUpstreamURL(scope, name, version, "Package.swift", "") - doc, err := h.fetchDocument(ctx, swiftManifestCacheKey(scope, name, version, ""), upstreamURL, swiftAcceptManifest) - if err != nil { - return err - } - for _, query := range swiftLinkQueries(doc.link, "alternate", upstreamURL) { - alternateURL := h.buildUpstreamURL(scope, name, version, "Package.swift", query) - if _, err := h.fetchDocument(ctx, swiftManifestCacheKey(scope, name, version, query), alternateURL, swiftAcceptManifest); err != nil { - return fmt.Errorf("%s: %w", query, err) - } - } - return nil -} - -// swiftLinkQueries returns the queries of the links with relation rel in -// link, a Link header, that point at requestURL's own path with a query, -// such as the next page of a release list or a version-specific manifest. -func swiftLinkQueries(link, rel, requestURL string) []string { - base, err := url.Parse(requestURL) - if err != nil { - return nil - } - var queries []string - for _, value := range strings.Split(link, ",") { - target, params, ok := strings.Cut(strings.TrimSpace(value), ";") - if !ok || !strings.HasPrefix(target, "<") || !strings.HasSuffix(target, ">") || !swiftLinkHasRel(params, rel) { - continue - } - ref, err := url.Parse(strings.TrimSpace(target[1 : len(target)-1])) - if err != nil { - continue - } - resolved := base.ResolveReference(ref) - if resolved.Host != base.Host || resolved.EscapedPath() != base.EscapedPath() || resolved.RawQuery == "" { - continue - } - queries = append(queries, resolved.RawQuery) - } - return queries -} - -func swiftLinkHasRel(params, rel string) bool { - for _, param := range strings.Split(params, ";") { - key, value, ok := strings.Cut(strings.TrimSpace(param), "=") - if !ok || !strings.EqualFold(strings.TrimSpace(key), "rel") { - continue - } - for _, r := range strings.Fields(strings.Trim(strings.TrimSpace(value), `"`)) { - if strings.EqualFold(r, rel) { - return true - } - } - } - return false -} - -// SwiftIdentity is a package's identity in a Swift registry, scope.name. -type SwiftIdentity struct { - Scope, Name string -} - -// ResolveIdentity returns the first registry identity the upstream lists for -// the source repository at sourceURL, the one a download uses. -func (h *SwiftHandler) ResolveIdentity(ctx context.Context, sourceURL string) (scope, name string, _ error) { - ids, err := h.ResolveIdentities(ctx, sourceURL) - if err != nil { - return "", "", err - } - return ids[0].Scope, ids[0].Name, nil -} - -// ResolveIdentities returns every registry identity the upstream lists for -// the source repository at sourceURL; the registry protocol allows several. -// The answer is cached as the identifiers route caches it. -func (h *SwiftHandler) ResolveIdentities(ctx context.Context, sourceURL string) ([]SwiftIdentity, error) { - query := url.Values{"url": {sourceURL}}.Encode() - body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "swift", swiftMetadataCacheKey("identifiers", query), - h.upstreamURL+"/identifiers?"+query, swiftAcceptJSON) - if errors.Is(err, ErrUpstreamNotFound) { - return nil, fmt.Errorf("the Swift registry has no package for %s", sourceURL) - } - if err != nil { - return nil, err - } - return SwiftIdentitiesOf(body, sourceURL) -} - -// SwiftIdentityOf returns the scope and name of the first identity in body, -// a registry's answer to an identifiers lookup for sourceURL. -func SwiftIdentityOf(body []byte, sourceURL string) (scope, name string, _ error) { - ids, err := SwiftIdentitiesOf(body, sourceURL) - if err != nil { - return "", "", err - } - return ids[0].Scope, ids[0].Name, nil -} - -// SwiftIdentitiesOf returns every identity in body, a registry's answer to -// an identifiers lookup for sourceURL. One invalid identity fails the whole -// answer, so a policy never covers only some of them. -func SwiftIdentitiesOf(body []byte, sourceURL string) ([]SwiftIdentity, error) { - var answer struct { - Identifiers []string `json:"identifiers"` - } - if err := json.Unmarshal(body, &answer); err != nil { - return nil, fmt.Errorf("parsing the Swift registry's identifiers for %s: %w", sourceURL, err) - } - if len(answer.Identifiers) == 0 { - return nil, fmt.Errorf("the Swift registry has no package for %s", sourceURL) - } - ids := make([]SwiftIdentity, 0, len(answer.Identifiers)) - for _, identifier := range answer.Identifiers { - scope, name, ok := strings.Cut(identifier, ".") - if !ok || scope == "" || name == "" { - return nil, fmt.Errorf("the Swift registry answered %s with the invalid identity %q", sourceURL, identifier) - } - ids = append(ids, SwiftIdentity{Scope: scope, Name: name}) - } - return ids, nil -} diff --git a/internal/handler/swift_mirror_test.go b/internal/handler/swift_mirror_test.go deleted file mode 100644 index 8359a5e..0000000 --- a/internal/handler/swift_mirror_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package handler - -import ( - "reflect" - "testing" -) - -func TestSwiftLinkQueries(t *testing.T) { - const manifest = "https://registry.example/swift/apple/example/1.0.0/Package.swift" - link := `; rel="alternate"; filename="Package@swift-5.8.swift", ` + - `; rel=alternate, ` + - `; rel="alternate", ` + - `; rel="alternate", ` + - `; rel="canonical"` - - got := swiftLinkQueries(link, "alternate", manifest) - - want := []string{"swift-version=5.8", "swift-version=5.7"} - if !reflect.DeepEqual(got, want) { - t.Errorf("swiftLinkQueries() = %q, want %q", got, want) - } -} diff --git a/internal/handler/swift_test.go b/internal/handler/swift_test.go deleted file mode 100644 index c53be77..0000000 --- a/internal/handler/swift_test.go +++ /dev/null @@ -1,585 +0,0 @@ -package handler - -import ( - "context" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries/fetch" -) - -func TestSwiftPackageReleasesRewritesRegistryURLs(t *testing.T) { - var gotAccept string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/registry/apple/swift-argument-parser" { - t.Errorf("upstream path = %q", r.URL.Path) - } - gotAccept = r.Header.Get("Accept") - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.Header().Set("Content-Version", "1") - w.Header().Add("Link", `; rel="next"`) - _, _ = io.WriteString(w, `{"releases":{"1.2.0":{"url":"/registry/apple/swift-argument-parser/1.2.0"},"1.1.0":{}}}`) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes() - req := httptest.NewRequest(http.MethodGet, "/APPLE/SWIFT-ARGUMENT-PARSER", nil) - req.Header.Set("Accept", swiftAcceptJSON) - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) - } - if gotAccept != swiftAcceptJSON { - t.Errorf("upstream Accept = %q, want %q", gotAccept, swiftAcceptJSON) - } - if got := w.Header().Get("Content-Version"); got != "1" { - t.Errorf("Content-Version = %q, want 1", got) - } - if got := w.Header().Get("Link"); got != `; rel="next"` { - t.Errorf("Link = %q", got) - } - - var body struct { - Releases map[string]struct { - URL string `json:"url"` - } `json:"releases"` - } - if err := json.NewDecoder(w.Body).Decode(&body); err != nil { - t.Fatalf("decoding response: %v", err) - } - if got := body.Releases["1.2.0"].URL; got != "https://proxy.example/swift/apple/swift-argument-parser/1.2.0" { - t.Errorf("release URL = %q", got) - } - if got := body.Releases["1.1.0"].URL; got != "" { - t.Errorf("release without upstream URL gained URL %q", got) - } -} - -func TestSwiftReleaseMetadataSupportsJSONExtensionAndHead(t *testing.T) { - var requestMethods []string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requestMethods = append(requestMethods, r.Method) - if r.URL.Path != "/registry/apple/example/1.2.3" { - t.Errorf("upstream path = %q", r.URL.Path) - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"id":"apple.example","version":"1.2.3","resources":[]}`) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes() - - for _, method := range []string{http.MethodGet, http.MethodHead} { - req := httptest.NewRequest(method, "/APPLE/EXAMPLE/1.2.3.json", nil) - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("%s status = %d, want 200", method, w.Code) - } - if method == http.MethodHead && w.Body.Len() != 0 { - t.Errorf("HEAD response body length = %d, want 0", w.Body.Len()) - } - } - if len(requestMethods) != 2 || requestMethods[0] != http.MethodGet || requestMethods[1] != http.MethodGet { - t.Errorf("upstream methods = %v, want metadata GETs", requestMethods) - } -} - -func TestSwiftManifestProxiesQueryAndRewritesLinks(t *testing.T) { - var upstream *httptest.Server - var gotAccept string - upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - t.Errorf("upstream method = %s, want GET", r.Method) - } - if r.URL.Path != "/registry/apple/example/1.2.3/Package.swift" { - t.Errorf("upstream path = %q", r.URL.Path) - } - if got := r.URL.Query().Get("swift-version"); got != "5.9" { - t.Errorf("swift-version = %q, want 5.9", got) - } - gotAccept = r.Header.Get("Accept") - w.Header().Set("Content-Type", "text/x-swift") - w.Header().Add("Link", fmt.Sprintf(`<%s/registry/apple/example/1.2.3/Package.swift?swift-version=5.8>; rel="alternate"; filename="Package@swift-5.8.swift"`, upstream.URL)) - w.Header().Add("Link", `; rel="canonical"`) - _, _ = io.WriteString(w, "// swift-tools-version: 5.9\n") - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes() - req := httptest.NewRequest(http.MethodGet, "/APPLE/EXAMPLE/1.2.3/Package.swift?swift-version=5.9", nil) - req.Header.Set("Accept", swiftAcceptManifest) - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200", w.Code) - } - if gotAccept != swiftAcceptManifest { - t.Errorf("upstream Accept = %q, want %q", gotAccept, swiftAcceptManifest) - } - links := strings.Join(w.Header().Values("Link"), ",") - if !strings.Contains(links, "https://proxy.example/swift/apple/example/1.2.3/Package.swift?swift-version=5.8") { - t.Errorf("internal manifest Link was not rewritten: %q", links) - } - if !strings.Contains(links, "https://github.com/apple/example") { - t.Errorf("external canonical Link was changed: %q", links) - } - if got := w.Header().Get("Content-Version"); got != "1" { - t.Errorf("Content-Version = %q, want 1", got) - } -} - -func TestSwiftSourceArchiveCachesAndPreservesSecurityMetadata(t *testing.T) { - archive := []byte("swift source archive") - checksumBytes := sha256.Sum256(archive) - checksum := hex.EncodeToString(checksumBytes[:]) - signature := base64.StdEncoding.EncodeToString([]byte("signature")) - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/registry/apple/example/1.2.3" { - t.Errorf("metadata path = %q", r.URL.Path) - } - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q,"signing":{"signatureBase64Encoded":%q,"signatureFormat":"cms-1.0.0"}}]}`, checksum, signature) - })) - defer upstream.Close() - - proxy, db, _, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader(string(archive))), - Size: int64(len(archive)), - ContentType: "application/zip", - } - handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes() - - requestArchive := func(method string) *httptest.ResponseRecorder { - req := httptest.NewRequest(method, "/apple/example/1.2.3.zip", nil) - req.Header.Set("Accept", swiftAcceptArchive) - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - return w - } - - w := requestArchive(http.MethodGet) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) - } - if got := w.Body.Bytes(); string(got) != string(archive) { - t.Errorf("archive body = %q", got) - } - if !fetcher.fetchCalled { - t.Fatal("archive fetcher was not called") - } - if got := fetcher.fetchedURL; got != upstream.URL+"/registry/apple/example/1.2.3.zip" { - t.Errorf("fetched URL = %q", got) - } - if got := fetcher.fetchedHeader.Get("Accept"); got != swiftAcceptArchive { - t.Errorf("archive Accept = %q, want %q", got, swiftAcceptArchive) - } - if got := w.Header().Get("Digest"); got != "sha-256="+base64.StdEncoding.EncodeToString(checksumBytes[:]) { - t.Errorf("Digest = %q", got) - } - if got := w.Header().Get("X-Swift-Package-Signature"); got != signature { - t.Errorf("signature = %q", got) - } - if got := w.Header().Get("X-Swift-Package-Signature-Format"); got != "cms-1.0.0" { - t.Errorf("signature format = %q", got) - } - if got := w.Header().Get("Content-Disposition"); got != `attachment; filename="example-1.2.3.zip"` { - t.Errorf("Content-Disposition = %q", got) - } - - packagePURL, versionPURL := packageurl.MakeCacheStrings("swift", "apple/example", "1.2.3") - if strings.HasPrefix(packagePURL, "pkg:swift/") { - t.Fatalf("registry identity produced source PURL %q", packagePURL) - } - versionRecord, err := db.GetVersionByPURL(versionPURL) - if err != nil { - t.Fatalf("cached Swift version %q not found: %v", versionPURL, err) - } - if versionRecord == nil { - t.Fatalf("cached Swift version %q not found", versionPURL) - } - if versionRecord.PackagePURL != packagePURL { - t.Errorf("cached package PURL = %q, want %q", versionRecord.PackagePURL, packagePURL) - } - - fetcher.fetchCalled = false - w = requestArchive(http.MethodHead) - if w.Code != http.StatusOK { - t.Fatalf("HEAD status = %d, want 200", w.Code) - } - if w.Body.Len() != 0 { - t.Errorf("HEAD body length = %d, want 0", w.Body.Len()) - } - if got := w.Header().Get("Content-Length"); got != fmt.Sprint(len(archive)) { - t.Errorf("HEAD Content-Length = %q", got) - } - - w = requestArchive(http.MethodGet) - if w.Code != http.StatusOK || w.Body.String() != string(archive) { - t.Fatalf("cached response = %d %q", w.Code, w.Body.Bytes()) - } - if fetcher.fetchCalled { - t.Error("cached archive contacted artifact upstream") - } -} - -func TestSwiftSourceArchiveRejectsChecksumMismatch(t *testing.T) { - archive := []byte("unexpected archive") - expectedChecksum := sha256.Sum256([]byte("expected archive")) - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(expectedChecksum[:])) - })) - defer upstream.Close() - - proxy, db, store, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader(string(archive))), - Size: int64(len(archive)), - ContentType: "application/zip", - } - handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes() - - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil)) - - if w.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want 502; body: %s", w.Code, w.Body.String()) - } - if len(store.files) != 0 { - t.Errorf("mismatched archive remained in storage: %v", store.files) - } - packagePURL, versionPURL := packageurl.MakeCacheStrings("swift", "apple/example", "1.2.3") - cached, err := db.GetCachedArtifact(packagePURL, versionPURL, "example-1.2.3.zip") - if err != nil { - t.Fatalf("checking cache: %v", err) - } - if cached != nil { - t.Error("mismatched archive gained a cache record") - } -} - -func TestSwiftSourceArchiveCanonicalizesPackageIdentity(t *testing.T) { - archive := []byte("swift source archive") - checksumBytes := sha256.Sum256(archive) - checksum := hex.EncodeToString(checksumBytes[:]) - var metadataPaths []string - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - metadataPaths = append(metadataPaths, r.URL.Path) - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, checksum) - })) - defer upstream.Close() - - proxy, db, store, fetcher := setupTestProxy(t) - handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes() - requestArchive := func(path string) { - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader(string(archive))), - Size: int64(len(archive)), - ContentType: "application/zip", - } - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) - if w.Code != http.StatusOK { - t.Fatalf("GET %s status = %d, want 200; body: %s", path, w.Code, w.Body.String()) - } - } - - requestArchive("/apple/example/1.2.3.zip") - requestArchive("/APPLE/EXAMPLE/1.2.3.zip") - - if len(store.files) != 1 { - t.Errorf("cached files = %d, want 1", len(store.files)) - } - for _, path := range metadataPaths { - if path != "/apple/example/1.2.3" { - t.Errorf("metadata path = %q, want canonical lowercase path", path) - } - } - - canonicalPURL, _ := packageurl.MakeCacheStrings("swift", "apple/example", "1.2.3") - canonical, err := db.GetPackageByPURL(canonicalPURL) - if err != nil { - t.Fatalf("getting canonical package: %v", err) - } - if canonical == nil { - t.Fatalf("canonical package %q not found", canonicalPURL) - } - - nonCanonicalPURL, _ := packageurl.MakeCacheStrings("swift", "APPLE/EXAMPLE", "1.2.3") - if nonCanonicalPURL != canonicalPURL { - t.Errorf("uppercase cache PURL = %q, want %q", nonCanonicalPURL, canonicalPURL) - } -} - -func TestSwiftSourceArchiveHeadLeavesStaleCacheForTheFetch(t *testing.T) { - archive := []byte("cached archive") - upstreamChecksum := sha256.Sum256([]byte("upstream archive")) - - var probed bool - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasSuffix(r.URL.Path, ".zip") { - probed = true - w.Header().Set("Content-Range", "bytes 0-0/456") - w.WriteHeader(http.StatusPartialContent) - _, _ = w.Write([]byte("x")) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(upstreamChecksum[:])) - })) - defer upstream.Close() - - proxy, db, store, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader(string(archive))), - Size: int64(len(archive)), - ContentType: "application/zip", - } - packagePURL, versionPURL := packageurl.MakeCacheStrings("swift", "apple/example", "1.2.3") - cached, err := proxy.getOrFetchArtifactFromURLWithCachePURLs( - context.Background(), "swift", "apple/example", "1.2.3", "example-1.2.3.zip", - packagePURL, versionPURL, upstream.URL+"/apple/example/1.2.3.zip", nil, "", "", - ) - if err != nil { - t.Fatalf("seeding cache: %v", err) - } - _ = cached.Reader.Close() - - handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes() - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/apple/example/1.2.3.zip", nil)) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) - } - if !probed { - t.Error("stale cache entry was not replaced by an upstream probe") - } - if got := w.Header().Get("Content-Length"); got != "456" { - t.Errorf("Content-Length = %q, want 456 from upstream probe", got) - } - // HEAD leaves the stale entry alone; the next GET replaces it under the - // coalescing key. - if len(store.files) != 1 { - t.Errorf("HEAD must leave the stale archive in storage, got %v", store.files) - } - if rec, _ := db.GetCachedArtifact(packagePURL, versionPURL, "example-1.2.3.zip"); rec == nil { - t.Error("HEAD must leave the stale cache record in place") - } - - fetcher.artifact = artifactBody("upstream archive") - w = httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil)) - if w.Code != http.StatusOK { - t.Fatalf("GET status = %d, want 200; body: %s", w.Code, w.Body.String()) - } - if w.Body.String() != "upstream archive" { - t.Errorf("GET body = %q, want the refreshed archive", w.Body.String()) - } - rec, _ := db.GetCachedArtifact(packagePURL, versionPURL, "example-1.2.3.zip") - if rec == nil || rec.Artifact.Digest.Encoded() != hex.EncodeToString(upstreamChecksum[:]) { - t.Errorf("cache record after GET = %+v, want the upstream checksum", rec) - } -} - -func TestSwiftSourceArchiveRequiresReleaseMetadata(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - })) - defer upstream.Close() - - proxy, _, store, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("signed archive")), - ContentType: "application/zip", - } - handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes() - - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil)) - - if w.Code != http.StatusBadGateway { - t.Fatalf("status = %d, want 502; body: %s", w.Code, w.Body.String()) - } - if fetcher.fetchCalled { - t.Error("archive was fetched without release security metadata") - } - if len(store.files) != 0 { - t.Errorf("archive was cached without release security metadata: %v", store.files) - } -} - -func TestSwiftSourceArchiveColdHeadUsesRangeGetAcrossRedirect(t *testing.T) { - checksum := strings.Repeat("a", sha256.Size*2) - var archiveAccept string - var archiveMethod string - var archiveRange string - download := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - archiveMethod = r.Method - archiveRange = r.Header.Get("Range") - w.Header().Set("Content-Range", "bytes 0-0/123") - w.WriteHeader(http.StatusPartialContent) - _, _ = w.Write([]byte("x")) - })) - defer download.Close() - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/apple/example/1.2.3": - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, checksum) - case "/apple/example/1.2.3.zip": - archiveAccept = r.Header.Get("Accept") - http.Redirect(w, r, download.URL, http.StatusSeeOther) - default: - http.NotFound(w, r) - } - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes() - - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/apple/example/1.2.3.zip", nil)) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) - } - if archiveMethod != http.MethodGet { - t.Errorf("download method = %q, want GET", archiveMethod) - } - if archiveRange != "bytes=0-0" { - t.Errorf("download Range = %q, want bytes=0-0", archiveRange) - } - if archiveAccept != swiftAcceptArchive { - t.Errorf("upstream Accept = %q, want %q", archiveAccept, swiftAcceptArchive) - } - if got := w.Header().Get("Content-Length"); got != "123" { - t.Errorf("Content-Length = %q, want 123", got) - } -} - -func TestSwiftIdentifiersAndPublishingUnsupported(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/registry/identifiers" { - t.Errorf("upstream path = %q", r.URL.Path) - } - if got := r.URL.Query().Get("url"); got != "https://github.com/apple/example" { - t.Errorf("lookup URL = %q", got) - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"identifiers":["apple.example"]}`) - })) - defer upstream.Close() - - proxy, _, _, _ := setupTestProxy(t) - handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes() - - req := httptest.NewRequest(http.MethodGet, "/identifiers?url=https%3A%2F%2Fgithub.com%2Fapple%2Fexample", nil) - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), "apple.example") { - t.Fatalf("identifier response = %d %q", w.Code, w.Body.String()) - } - - req = httptest.NewRequest(http.MethodGet, "/identifiers", nil) - w = httptest.NewRecorder() - handler.ServeHTTP(w, req) - if w.Code != http.StatusBadRequest { - t.Errorf("missing URL status = %d, want 400", w.Code) - } - - req = httptest.NewRequest(http.MethodPut, "/apple/example/1.2.3", strings.NewReader("ignored")) - w = httptest.NewRecorder() - handler.ServeHTTP(w, req) - if w.Code != http.StatusMethodNotAllowed { - t.Errorf("publish status = %d, want 405", w.Code) - } - if got := w.Header().Get("Allow"); got != "GET, HEAD" { - t.Errorf("Allow = %q", got) - } -} - -func TestSwiftIdentifierValidation(t *testing.T) { - tests := []struct { - name string - value string - valid func(string) bool - want bool - }{ - {"scope", "apple", validSwiftScope, true}, - {"scope hyphen", "swift-server", validSwiftScope, true}, - {"scope underscore", "swift_server", validSwiftScope, false}, - {"scope repeated separator", "swift--server", validSwiftScope, false}, - {"package", "swift-argument_parser", validSwiftPackageName, true}, - {"package repeated separators", "swift-_argument", validSwiftPackageName, false}, - {"package trailing separator", "example-", validSwiftPackageName, false}, - {"package non-ASCII", "café", validSwiftPackageName, false}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if got := test.valid(test.value); got != test.want { - t.Errorf("validation of %q = %v, want %v", test.value, got, test.want) - } - }) - } -} - -func TestSwiftRetentionKey(t *testing.T) { - tests := []struct { - key string - want string - ok bool - }{ - {"pkg:swift/apple/swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true}, - {"pkg:swift/Apple/Swift-NIO", "pkg:generic/swift-registry/apple.swift-nio", true}, - {"pkg:generic/swift-registry/apple.swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true}, - {"pkg:generic/swift-registry/Apple.Swift-NIO", "pkg:generic/swift-registry/apple.swift-nio", true}, - {"pkg:generic/Swift-Registry/apple.swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true}, - {"pkg:swift/github.com/apple/swift-nio", "", false}, - {"pkg:generic/swift-registry/swift-nio", "", false}, - {"pkg:generic/swift-registry/apple.swift.nio", "", false}, - {"pkg:swift/apple/-nio", "", false}, - {"pkg:generic/apple.swift-nio", "", false}, - {"pkg:generic/other/apple.swift-nio", "", false}, - {"pkg:npm/apple/swift-nio", "", false}, - } - for _, tt := range tests { - p, err := purl.Parse(tt.key) - if err != nil { - t.Fatalf("Parse(%s): %v", tt.key, err) - } - got, ok := swiftRetentionKey(p) - if ok != tt.ok || (ok && got != tt.want) { - t.Errorf("swiftRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) - } - } - if _, ok := swiftRetentionKey(nil); ok { - t.Error("swiftRetentionKey(nil) accepted") - } -} diff --git a/internal/handler/unrecorded_test.go b/internal/handler/unrecorded_test.go deleted file mode 100644 index d87ddc9..0000000 --- a/internal/handler/unrecorded_test.go +++ /dev/null @@ -1,415 +0,0 @@ -package handler - -import ( - "context" - "errors" - "io" - "maps" - "net/http" - "net/http/httptest" - "slices" - "sync" - "testing" - "time" -) - -// These tests refetch an artifact whose stored file is gone while -// pending_deletes refuses writes. The record can't move to the new file and -// the file can't be queued, so the fetch must serve it to every caller sharing -// it and delete it once the last of them is done. - -const ( - unrecordedContent = "refetched content" - unrecordedURL = "https://example.com/gone.tgz" -) - -func setupUnrecorded(t *testing.T) (*Proxy, *mockStorage) { - t.Helper() - proxy, db, store, _ := setupTestProxy(t) - seedPackage(t, db, store, "npm", "gone", "1.0.0", "gone-1.0.0.tgz", "old content") - oldPath := recordedStoragePath(t, db, "pkg:npm/gone@1.0.0", "gone-1.0.0.tgz") - delete(store.files, oldPath) - if _, err := db.Exec(`CREATE TRIGGER refuse_queue BEFORE INSERT ON pending_deletes - BEGIN SELECT RAISE(ABORT, 'queue refused'); END`); err != nil { - t.Fatalf("creating trigger: %v", err) - } - t.Cleanup(func() { - if got := recordedStoragePath(t, db, "pkg:npm/gone@1.0.0", "gone-1.0.0.tgz"); got != oldPath { - t.Errorf("record points at %q, want it kept at %q", got, oldPath) - } - }) - return proxy, store -} - -func fetchGone(ctx context.Context, proxy *Proxy) (*CacheResult, error) { - return proxy.GetOrFetchArtifactFromURL(ctx, "npm", "gone", "1.0.0", "gone-1.0.0.tgz", unrecordedURL) -} - -func storedFiles(store *mockStorage) []string { - store.mu.Lock() - defer store.mu.Unlock() - return slices.Collect(maps.Keys(store.files)) -} - -func readAndClose(t *testing.T, res *CacheResult) { - t.Helper() - body, err := io.ReadAll(res.Reader) - _ = res.Reader.Close() - if err != nil || string(body) != unrecordedContent { - t.Errorf("body = %q, %v, want %q", body, err, unrecordedContent) - } -} - -// waitForWaiters blocks until n callers wait on the one fetch in flight. -func waitForWaiters(t *testing.T, proxy *Proxy, n int) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - proxy.fetchMu.Lock() - got := -1 - for _, f := range proxy.inFlight { - got = f.waiters - } - proxy.fetchMu.Unlock() - if got == n { - return - } - time.Sleep(time.Millisecond) - } - t.Fatalf("never saw %d waiters on the fetch", n) -} - -func TestUnrecordedRefetchIsDeletedOnceServed(t *testing.T) { - proxy, store := setupUnrecorded(t) - fetcher := &countingFetcher{content: unrecordedContent} - proxy.Fetcher = fetcher - - for range 2 { - res, err := fetchGone(context.Background(), proxy) - if err != nil { - t.Fatalf("fetch: %v", err) - } - if got := storedFiles(store); len(got) != 1 { - t.Errorf("stored files while serving = %v, want the refetched one", got) - } - readAndClose(t, res) - if got := storedFiles(store); len(got) != 0 { - t.Errorf("stored files after serving = %v, want none left untracked", got) - } - } -} - -func TestUnrecordedRefetchServesRangeRequests(t *testing.T) { - proxy, store := setupUnrecorded(t) - store.seekable = true - proxy.Fetcher = &countingFetcher{content: unrecordedContent} - - res, err := fetchGone(context.Background(), proxy) - if err != nil { - t.Fatalf("fetch: %v", err) - } - req := httptest.NewRequest(http.MethodGet, "/gone-1.0.0.tgz", nil) - req.Header.Set("Range", "bytes=2-5") - rec := httptest.NewRecorder() - ServeArtifactRequest(rec, req, res) - - if rec.Code != http.StatusPartialContent || rec.Body.String() != unrecordedContent[2:6] { - t.Errorf("range response = %d %q, want 206 %q", rec.Code, rec.Body.String(), unrecordedContent[2:6]) - } - if got := storedFiles(store); len(got) != 0 { - t.Errorf("stored files after serving = %v, want none left untracked", got) - } -} - -// TestUnrecordedRefetchWaitsForEveryCaller keeps the object until the caller -// that ran the fetch and every caller that waited on it have closed. -func TestUnrecordedRefetchWaitsForEveryCaller(t *testing.T) { - const waiters = 4 - proxy, store := setupUnrecorded(t) - fetcher := newGatedFetcher(unrecordedContent) - proxy.Fetcher = fetcher - - leader := make(chan *CacheResult, 1) - go func() { - res, err := fetchGone(context.Background(), proxy) - if err != nil { - t.Errorf("leader: %v", err) - } - leader <- res - }() - <-fetcher.entered - - results := make(chan *CacheResult, waiters) - var wg sync.WaitGroup - for range waiters { - wg.Go(func() { - res, err := fetchGone(context.Background(), proxy) - if err != nil { - t.Errorf("waiter: %v", err) - return - } - results <- res - }) - } - waitForWaiters(t, proxy, waiters) - close(fetcher.release) - - // The leader holds its reader open while every waiter reads and closes. - lead := <-leader - wg.Wait() - close(results) - for res := range results { - readAndClose(t, res) - } - if got := storedFiles(store); len(got) != 1 { - t.Fatalf("stored files with the leader still reading = %v, want the refetched one", got) - } - readAndClose(t, lead) - if got := storedFiles(store); len(got) != 0 { - t.Errorf("stored files after every caller closed = %v, want none", got) - } -} - -func TestUnrecordedRefetchSurvivesDoubleClose(t *testing.T) { - proxy, store := setupUnrecorded(t) - fetcher := newGatedFetcher(unrecordedContent) - proxy.Fetcher = fetcher - - leader := make(chan *CacheResult, 1) - go func() { - res, err := fetchGone(context.Background(), proxy) - if err != nil { - t.Errorf("leader: %v", err) - } - leader <- res - }() - <-fetcher.entered - - waiter := make(chan *CacheResult, 1) - go func() { - res, err := fetchGone(context.Background(), proxy) - if err != nil { - t.Errorf("waiter: %v", err) - } - waiter <- res - }() - waitForWaiters(t, proxy, 1) - close(fetcher.release) - - lead, wait := <-leader, <-waiter - readAndClose(t, lead) - _ = lead.Reader.Close() - if got := storedFiles(store); len(got) != 1 { - t.Fatalf("stored files with the waiter still reading = %v, want the refetched one", got) - } - readAndClose(t, wait) - if got := storedFiles(store); len(got) != 0 { - t.Errorf("stored files after every caller closed = %v, want none", got) - } -} - -// ctxStorage fails to open under a cancelled context, as cloud backends do. -type ctxStorage struct{ *mockStorage } - -func (s ctxStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - return s.mockStorage.Open(ctx, path) -} - -// TestUnrecordedRefetchOutlivesDetachedCaller is the retry a detached fetch -// exists for: the client that started it gave up, and its retry waits on the -// same fetch. The retry must get the bytes rather than a deleted object. -func TestUnrecordedRefetchOutlivesDetachedCaller(t *testing.T) { - proxy, store := setupUnrecorded(t) - proxy.Storage = ctxStorage{store} - fetcher := newGatedFetcher(unrecordedContent) - proxy.Fetcher = fetcher - - ctx, cancel := context.WithCancel(DetachUpstreamFetch(context.Background())) - leaderDone := make(chan struct{}) - go func() { - defer close(leaderDone) - res, _ := fetchGone(ctx, proxy) - drain(res) - }() - <-fetcher.entered - cancel() - - retry := make(chan *CacheResult, 1) - go func() { - res, err := fetchGone(context.Background(), proxy) - if err != nil { - t.Errorf("retry: %v", err) - } - retry <- res - }() - waitForWaiters(t, proxy, 1) - close(fetcher.release) - <-leaderDone - - if res := <-retry; res != nil { - readAndClose(t, res) - } - if got := storedFiles(store); len(got) != 0 { - t.Errorf("stored files after the retry closed = %v, want none", got) - } -} - -// TestUnrecordedRefetchIgnoresWaiterThatLeft checks that a waiter giving up -// before the fetch ends does not keep the object forever. -func TestUnrecordedRefetchIgnoresWaiterThatLeft(t *testing.T) { - proxy, store := setupUnrecorded(t) - fetcher := newGatedFetcher(unrecordedContent) - proxy.Fetcher = fetcher - - leader := make(chan *CacheResult, 1) - go func() { - res, err := fetchGone(context.Background(), proxy) - if err != nil { - t.Errorf("leader: %v", err) - } - leader <- res - }() - <-fetcher.entered - - ctx, cancel := context.WithCancel(context.Background()) - left := make(chan error, 1) - go func() { - _, err := fetchGone(ctx, proxy) - left <- err - }() - waitForWaiters(t, proxy, 1) - cancel() - if err := <-left; !errors.Is(err, context.Canceled) { - t.Fatalf("waiter that left: err = %v, want context.Canceled", err) - } - close(fetcher.release) - - readAndClose(t, <-leader) - if got := storedFiles(store); len(got) != 0 { - t.Errorf("stored files after the leader closed = %v, want none", got) - } -} - -// TestLeaveFetch covers both ways a waiter can give up: before the fetch -// finished, when it was not handed a hold yet, and after, when it was. -func TestLeaveFetch(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - deleted := false - newFetch := func() *inflightFetch { - f := &inflightFetch{done: make(chan struct{}), waiters: 1} - f.unrecorded = &unrecordedObject{delete: func() { deleted = true }} - f.unrecorded.refs.Add(1) - return f - } - - f := newFetch() - proxy.leaveFetch(f) - if f.waiters != 0 || deleted { - t.Errorf("before done: waiters = %d, deleted = %v, want 0 and false", f.waiters, deleted) - } - - f = newFetch() - close(f.done) - proxy.leaveFetch(f) - if f.waiters != 1 || !deleted { - t.Errorf("after done: waiters = %d, deleted = %v, want 1 and true", f.waiters, deleted) - } -} - -// TestRecordedAt is the check that keeps an object whose record update -// committed despite reporting an error. -func TestRecordedAt(t *testing.T) { - proxy, db, store, _ := setupTestProxy(t) - seedPackage(t, db, store, "npm", "kept", "1.0.0", "kept-1.0.0.tgz", "content") - path := recordedStoragePath(t, db, "pkg:npm/kept@1.0.0", "kept-1.0.0.tgz") - - if !proxy.recordedAt("pkg:npm/kept@1.0.0", "kept-1.0.0.tgz", path) { - t.Error("record pointing at the path not seen") - } - if proxy.recordedAt("pkg:npm/kept@1.0.0", "kept-1.0.0.tgz", "other/path.tgz") { - t.Error("record pointing elsewhere taken as recorded") - } - if proxy.recordedAt("pkg:npm/none@1.0.0", "none-1.0.0.tgz", path) { - t.Error("missing record taken as recorded") - } - _ = db.Close() - if proxy.recordedAt("pkg:npm/kept@1.0.0", "kept-1.0.0.tgz", path) { - t.Error("failed lookup taken as recorded") - } -} - -// TestUnrecordedStreamedMiss streams an unrecorded refetch to the caller -// that started it, and the waiters still get the stored object. A caller fed -// from upstream holds no reader on it, so it is deleted once the waiters are -// done; one that fell behind reads it, so it is deleted once that caller is -// done too. -func TestUnrecordedStreamedMiss(t *testing.T) { - for _, behind := range []bool{false, true} { - t.Run(map[bool]string{false: "fed", true: "behind"}[behind], func(t *testing.T) { - testUnrecordedStreamedMiss(t, behind) - }) - } -} - -func testUnrecordedStreamedMiss(t *testing.T, behind bool) { - t.Helper() - if behind { - old := clientFeedLimit - clientFeedLimit = 4 - t.Cleanup(func() { clientFeedLimit = old }) - } - proxy, store := setupUnrecorded(t) - fetcher := newGatedFetcher(unrecordedContent) - proxy.Fetcher = fetcher - - leader := make(chan *CacheResult, 1) - go func() { - res, err := fetchGone(StreamCacheMiss(context.Background()), proxy) - if err != nil { - t.Errorf("leader: %v", err) - } - leader <- res - }() - <-fetcher.entered - waiter := make(chan *CacheResult, 1) - go func() { - res, err := fetchGone(context.Background(), proxy) - if err != nil { - t.Errorf("waiter: %v", err) - } - waiter <- res - }() - waitForWaiters(t, proxy, 1) - close(fetcher.release) - - lead := <-leader - if lead == nil || lead.Artifact.Size != -1 { - t.Fatalf("leader = %+v, want a stream", lead) - } - w := <-waiter - if w == nil { - t.FailNow() - } - readAndClose(t, w) - if behind { - // The leader reads the stored copy, so it must outlive the - // waiter. - if len(storedFiles(store)) == 0 { - t.Fatal("stored copy deleted while the leader still reads it") - } - readAndClose(t, lead) - } - deadline := time.Now().Add(5 * time.Second) - for len(storedFiles(store)) != 0 && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if got := storedFiles(store); len(got) != 0 { - t.Errorf("stored files after every reader closed = %v, want none", got) - } - if !behind { - readAndClose(t, lead) - } -} diff --git a/internal/handler/upstream_auth_helper_test.go b/internal/handler/upstream_auth_helper_test.go deleted file mode 100644 index 261636d..0000000 --- a/internal/handler/upstream_auth_helper_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package handler - -import ( - "net/http" - - upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" -) - -// authorizeUpstream sends authorization on every upstream request the way the -// server does: through the auth transport, which scopes it to each hop. -func authorizeUpstream(proxy *Proxy, authorization string) { - base := proxy.HTTPClient.Transport - if base == nil { - base = http.DefaultTransport - } - client := *proxy.HTTPClient - client.Transport = upstreamhttp.NewTransport(base, func(string) (string, string) { - return "Authorization", authorization - }) - proxy.HTTPClient = &client -} diff --git a/internal/handler/upstream_body.go b/internal/handler/upstream_body.go deleted file mode 100644 index 7db9076..0000000 --- a/internal/handler/upstream_body.go +++ /dev/null @@ -1,26 +0,0 @@ -package handler - -import ( - "errors" - "io" -) - -// upstreamBodyReader marks a failure to read a document from upstream, so a -// body upstream cuts off is told apart from one that ends, and from a -// failure to store it. -type upstreamBodyReader struct{ r io.Reader } - -func (u upstreamBodyReader) Read(b []byte) (int, error) { - n, err := u.r.Read(b) - if err != nil && !errors.Is(err, io.EOF) { - err = &upstreamBodyError{err} - } - return n, err -} - -// upstreamBodyError is a failure to read a document from upstream. It is -// handled as an upstream failure: the stored copy, if any, is served. -type upstreamBodyError struct{ err error } - -func (e *upstreamBodyError) Error() string { return "reading upstream metadata: " + e.err.Error() } -func (e *upstreamBodyError) Unwrap() error { return e.err } diff --git a/internal/handler/upstream_held.go b/internal/handler/upstream_held.go deleted file mode 100644 index 57ac84d..0000000 --- a/internal/handler/upstream_held.go +++ /dev/null @@ -1,53 +0,0 @@ -package handler - -import ( - "errors" - "math" - "net/http" - "strconv" - - registryclient "github.com/git-pkgs/registries/client" -) - -// writeUpstreamHeld answers a request the rate limit gate refused to send -// with 503 and the Retry-After the upstream asked for, and reports whether -// err was such a refusal. -func writeUpstreamHeld(w http.ResponseWriter, err error) bool { - var held *registryclient.HeldError - if !errors.As(err, &held) { - return false - } - secs := int(math.Ceil(held.RetryAfter.Seconds())) - if secs < 1 { - secs = 1 - } - w.Header().Set("Retry-After", strconv.Itoa(secs)) - http.Error(w, held.Error(), http.StatusServiceUnavailable) - return true -} - -// UpstreamHolds reports whether requests to a URL's host are held by rate -// limiting. *registryclient.RateLimitGate and the shared gate in -// internal/httpclient implement it. -type UpstreamHolds interface { - // Held returns a *registryclient.HeldError while rawURL's host is held. - Held(rawURL string) error -} - -// upstreamHeld returns the hold on rawURL's host, if any. -func (p *Proxy) upstreamHeld(rawURL string) error { - if p.RateLimits == nil { - return nil - } - return p.RateLimits.Held(rawURL) -} - -// reloadUpstreamHolds rereads holds shared with other instances, when the -// gate shares them. It is called after another instance's fetch failed, -// since a 429 there will have placed a hold this instance may not have read -// yet. -func (p *Proxy) reloadUpstreamHolds() { - if r, ok := p.RateLimits.(interface{ Reload() }); ok { - r.Reload() - } -} diff --git a/internal/handler/upstream_test.go b/internal/handler/upstream_test.go deleted file mode 100644 index 9d99214..0000000 --- a/internal/handler/upstream_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package handler - -import ( - "context" - "io" - "net/http" - "net/http/httptest" - "testing" -) - -func TestHandlerUpstreamConfiguration(t *testing.T) { - const ( - proxyURL = "https://proxy.example.com/" - baseURL = "https://upstream.example.com" - ) - hex := NewHexHandlerWithUpstreams(nil, proxyURL, baseURL+"/hex/", baseURL+"/hex-api/") - pypi := NewPyPIHandlerWithUpstreams(nil, proxyURL, baseURL+"/pypi/", baseURL+"/pypi-download/") - nuget := NewNuGetHandlerWithUpstreams(nil, proxyURL, baseURL+"/nuget/", baseURL+"/nuget-search/") - composer := NewComposerHandlerWithUpstreams( - nil, proxyURL, baseURL+"/composer/", baseURL+"/composer-repository/", - ) - - got := map[string]string{ - "gem": NewGemHandlerWithUpstream(nil, proxyURL, baseURL+"/gem/").upstreamURL, - "go": NewGoHandlerWithUpstream(nil, proxyURL, baseURL+"/go/").upstreamURL, - "hex": hex.upstreamURL, - "hex_api": hex.apiURL, - "pub": NewPubHandlerWithUpstream(nil, proxyURL, baseURL+"/pub/").upstreamURL, - "pypi": pypi.upstreamURL, - "pypi_download": pypi.downloadURL, - "nuget": nuget.upstreamURL, - "nuget_search": nuget.searchURL, - "composer": composer.upstreamURL, - "composer_repository": composer.repoURL, - "conan": NewConanHandlerWithUpstream(nil, proxyURL, baseURL+"/conan/").upstreamURL, - "conda": NewCondaHandlerWithUpstream(nil, proxyURL, baseURL+"/conda/").upstreamURL, - "cran": NewCRANHandlerWithUpstream(nil, proxyURL, baseURL+"/cran/").upstreamURL, - "julia": NewJuliaHandlerWithUpstream(nil, baseURL+"/julia/").upstreamURL, - "oci_default": NewContainerHandlerWithRegistry(nil, proxyURL, baseURL+"/oci/").registryURL, - "rpm": NewRPMHandlerWithUpstream(nil, proxyURL, baseURL+"/rpm/").upstreamURL, - } - - want := map[string]string{ - "gem": baseURL + "/gem", - "go": baseURL + "/go", - "hex": baseURL + "/hex", - "hex_api": baseURL + "/hex-api", - "pub": baseURL + "/pub", - "pypi": baseURL + "/pypi", - "pypi_download": baseURL + "/pypi-download", - "nuget": baseURL + "/nuget", - "nuget_search": baseURL + "/nuget-search", - "composer": baseURL + "/composer", - "composer_repository": baseURL + "/composer-repository", - "conan": baseURL + "/conan", - "conda": baseURL + "/conda", - "cran": baseURL + "/cran", - "julia": baseURL + "/julia", - "oci_default": baseURL + "/oci", - "rpm": baseURL + "/rpm", - } - - for name, wantURL := range want { - if gotURL := got[name]; gotURL != wantURL { - t.Errorf("%s upstream = %q, want %q", name, gotURL, wantURL) - } - } -} - -func TestConfiguredUpstreamURL(t *testing.T) { - if got := configuredUpstreamURL("", "https://default.example.com/"); got != "https://default.example.com" { - t.Errorf("empty configured URL = %q, want default", got) - } - if got := configuredUpstreamURL("https://custom.example.com/", "https://default.example.com"); got != "https://custom.example.com" { - t.Errorf("configured URL = %q, want trimmed custom URL", got) - } - if got := configuredUpstreamURL("https://custom.example.com///", "https://default.example.com"); got != "https://custom.example.com" { - t.Errorf("configured URL with trailing slashes = %q, want trimmed custom URL", got) - } -} - -func TestHexHandlerUsesConfiguredAPIUpstream(t *testing.T) { - var requestedPath string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requestedPath = r.URL.Path - _, _ = io.WriteString(w, `{"releases":[]}`) - })) - defer upstream.Close() - - h := NewHexHandlerWithUpstreams( - &Proxy{HTTPClient: upstream.Client()}, - "https://proxy.example.com", - upstream.URL+"/hex", - upstream.URL+"/hex-api", - ) - timed := h.timedPackage("demo") - res := h.proxy.lookupMetadata(context.Background(), "hex", timed.timesKey, timed.timesURL, timed.timesAccept, "", nil) - if res.err != nil { - t.Fatalf("fetching publish times failed: %v", res.err) - } - if requestedPath != "/hex-api/api/packages/demo" { - t.Errorf("API path = %q, want %q", requestedPath, "/hex-api/api/packages/demo") - } -} - -func TestPyPIHandlerRewritesConfiguredDownloadUpstream(t *testing.T) { - h := NewPyPIHandlerWithUpstreams( - nil, - "https://proxy.example.com", - "https://upstream.example.com/pypi", - "https://upstream.example.com/pypi", - ) - body := []byte(`demo`) - want := `demo` - if got := string(h.rewriteSimpleHTML(body, nil)); got != want { - t.Errorf("rewritten HTML = %q, want %q", got, want) - } -} - -func TestNuGetHandlerUsesConfiguredSearchUpstream(t *testing.T) { - h := NewNuGetHandlerWithUpstreams( - nil, - "https://proxy.example.com", - "https://upstream.example.com/nuget", - "https://upstream.example.com/nuget-search", - ) - req := httptest.NewRequest(http.MethodGet, "/query?q=demo", nil) - want := "https://upstream.example.com/nuget-search/query?q=demo" - if got := h.buildUpstreamURL(req); got != want { - t.Errorf("search URL = %q, want %q", got, want) - } -} diff --git a/internal/httpclient/access_log.go b/internal/httpclient/access_log.go deleted file mode 100644 index 8e13f23..0000000 --- a/internal/httpclient/access_log.go +++ /dev/null @@ -1,74 +0,0 @@ -package httpclient - -import ( - "log/slog" - "net/http" - "net/url" - "strings" - "time" - - "github.com/git-pkgs/proxy/internal/accesslog" -) - -type accessLogTransport struct { - base http.RoundTripper - accessLog *accesslog.Logger - logger *slog.Logger -} - -// NewAccessLogTransport records each upstream HTTP exchange around base. -func NewAccessLogTransport(base http.RoundTripper, log *accesslog.Logger, logger *slog.Logger) http.RoundTripper { - if base == nil { - base = http.DefaultTransport - } - if logger == nil { - logger = slog.Default() - } - if log == nil { - return base - } - return &accessLogTransport{ - base: base, - accessLog: log, - logger: logger, - } -} - -func (t *accessLogTransport) RoundTrip(req *http.Request) (*http.Response, error) { - start := time.Now() - resp, err := t.base.RoundTrip(req) - - entry := accesslog.Entry{ - Event: accesslog.EventUpstream, - RequestID: accesslog.RequestID(req.Context()), - Method: req.Method, - URL: accesslog.URLWithoutSecrets(req.URL), - DurationMS: time.Since(start).Milliseconds(), - } - if resp != nil { - entry.StatusCode = resp.StatusCode - } - if err != nil { - entry.Error = errorWithoutSecrets(err, req.URL) - } - if writeErr := t.accessLog.Write(entry); writeErr != nil { - t.logger.Error("failed to write access log", "error", writeErr) - } - - return resp, err -} - -func errorWithoutSecrets(err error, requestURL *url.URL) string { - message := err.Error() - if requestURL == nil { - return message - } - - cleanURL := accesslog.URLWithoutSecrets(requestURL) - for _, value := range []string{requestURL.String(), requestURL.Redacted()} { - if value != "" { - message = strings.ReplaceAll(message, value, cleanURL) - } - } - return message -} diff --git a/internal/httpclient/access_log_test.go b/internal/httpclient/access_log_test.go deleted file mode 100644 index aa3a43c..0000000 --- a/internal/httpclient/access_log_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package httpclient - -import ( - "bufio" - "encoding/json" - "errors" - "io" - "log/slog" - "net/http" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/git-pkgs/proxy/internal/accesslog" -) - -type roundTripFunc func(*http.Request) (*http.Response, error) - -func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { - return f(req) -} - -func TestAccessLogTransportRecordsUpstreamStatus(t *testing.T) { - path := filepath.Join(t.TempDir(), "access.jsonl") - accessLogger, err := accesslog.Open(path) - if err != nil { - t.Fatal(err) - } - - base := roundTripFunc(func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: http.StatusTooManyRequests, - Body: io.NopCloser(strings.NewReader("rate limited")), - Request: req, - }, nil - }) - client := &http.Client{Transport: NewAccessLogTransport(base, accessLogger, slog.Default())} - req, err := http.NewRequest(http.MethodGet, "https://user:password@registry.example/package.tgz?token=secret", nil) - if err != nil { - t.Fatal(err) - } - req = req.WithContext(accesslog.WithRequestID(req.Context(), "request-123")) - - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - _ = resp.Body.Close() - if err := accessLogger.Close(); err != nil { - t.Fatal(err) - } - - entry := readAccessLogEntry(t, path) - if entry.Event != accesslog.EventUpstream { - t.Errorf("event = %q, want %q", entry.Event, accesslog.EventUpstream) - } - if entry.RequestID != "request-123" { - t.Errorf("request_id = %q, want %q", entry.RequestID, "request-123") - } - if entry.StatusCode != http.StatusTooManyRequests { - t.Errorf("status_code = %d, want %d", entry.StatusCode, http.StatusTooManyRequests) - } - if entry.URL != "https://registry.example/package.tgz" { - t.Errorf("url = %q, want URL without credentials or query", entry.URL) - } -} - -func TestAccessLogTransportRecordsUpstreamError(t *testing.T) { - path := filepath.Join(t.TempDir(), "access.jsonl") - accessLogger, err := accesslog.Open(path) - if err != nil { - t.Fatal(err) - } - - wantErr := errors.New("GET https://user:password@registry.example/package.tgz?token=secret: connection refused") - base := roundTripFunc(func(*http.Request) (*http.Response, error) { - return nil, wantErr - }) - client := &http.Client{Transport: NewAccessLogTransport(base, accessLogger, slog.Default())} - - _, err = client.Get("https://user:password@registry.example/package.tgz?token=secret") - if !errors.Is(err, wantErr) { - t.Fatalf("GET error = %v, want %v", err, wantErr) - } - if err := accessLogger.Close(); err != nil { - t.Fatal(err) - } - - entry := readAccessLogEntry(t, path) - if entry.StatusCode != 0 { - t.Errorf("status_code = %d, want 0", entry.StatusCode) - } - if strings.Contains(entry.Error, "password") || strings.Contains(entry.Error, "secret") { - t.Errorf("error contains URL credentials or query: %q", entry.Error) - } - if !strings.Contains(entry.Error, "connection refused") { - t.Errorf("error = %q, want connection failure", entry.Error) - } -} - -func readAccessLogEntry(t *testing.T, path string) accesslog.Entry { - t.Helper() - - file, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer func() { _ = file.Close() }() - - scanner := bufio.NewScanner(file) - if !scanner.Scan() { - t.Fatalf("access log is empty: %v", scanner.Err()) - } - - var entry accesslog.Entry - if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil { - t.Fatalf("decoding access log: %v", err) - } - return entry -} diff --git a/internal/httpclient/shared_holds.go b/internal/httpclient/shared_holds.go deleted file mode 100644 index 175931b..0000000 --- a/internal/httpclient/shared_holds.go +++ /dev/null @@ -1,213 +0,0 @@ -package httpclient - -import ( - "log/slog" - "net/http" - "net/url" - "strings" - "sync" - "time" - - registryclient "github.com/git-pkgs/registries/client" -) - -// DefaultHoldRefresh is how long a SharedRateLimitGate trusts its copy of -// the shared holds before reading them again. It bounds both the database -// reads, at one per interval per instance, and how long another instance -// may keep sending after a hold is placed. -const DefaultHoldRefresh = 2 * time.Second - -// HoldStore keeps rate-limit holds where every instance sharing it sees them. -type HoldStore interface { - PutUpstreamHold(host string, until time.Time) error - UpstreamHolds(now time.Time) (map[string]time.Time, error) -} - -// SharedRateLimitGate adds holds shared with other instances to a -// RateLimitGate. Holds the gate places are published to the store, and -// requests to a host another instance holds are refused like the gate's own, -// with a HeldError. Without a store it behaves as the gate alone. -type SharedRateLimitGate struct { - gate *registryclient.RateLimitGate - store HoldStore - logger *slog.Logger - now func() time.Time - - // Refresh is how long the shared holds read from the store are used - // before reading them again. Set it before use. - Refresh time.Duration - - mu sync.Mutex - shared map[string]time.Time - // published holds the holds this instance wrote, so a read that started - // before a write does not drop it. - published map[string]time.Time - loaded time.Time - loading bool -} - -// NewSharedRateLimitGate wraps gate, publishing its holds to store and -// honouring those in store. It takes over gate.OnHold, calling the callback -// set before. A nil store shares nothing. -func NewSharedRateLimitGate(gate *registryclient.RateLimitGate, store HoldStore, logger *slog.Logger) *SharedRateLimitGate { - s := &SharedRateLimitGate{ - gate: gate, - store: store, - logger: logger, - now: time.Now, - Refresh: DefaultHoldRefresh, - shared: make(map[string]time.Time), - published: make(map[string]time.Time), - } - if store != nil { - onHold := gate.OnHold - gate.OnHold = func(event registryclient.HoldEvent) { - s.publish(event.Host, event.Until) - if onHold != nil { - onHold(event) - } - } - } - return s -} - -// RoundTrip implements http.RoundTripper. -func (s *SharedRateLimitGate) RoundTrip(req *http.Request) (*http.Response, error) { - if err := s.sharedHeld(hostKey(req.URL)); err != nil { - return nil, err - } - return s.gate.RoundTrip(req) -} - -// Held returns a *registryclient.HeldError when requests to rawURL's host are -// held by this instance or another, and nil otherwise. It is safe to call on -// a nil gate. -func (s *SharedRateLimitGate) Held(rawURL string) error { - if s == nil { - return nil - } - if err := s.gate.Held(rawURL); err != nil { - return err - } - u, err := url.Parse(rawURL) - if err != nil { - return nil - } - return s.sharedHeld(hostKey(u)) -} - -// Holds returns the hosts held by this instance or another, and when each -// hold ends. It is safe to call on a nil gate. -func (s *SharedRateLimitGate) Holds() map[string]time.Time { - if s == nil { - return nil - } - holds := s.gate.Holds() - if s.store == nil { - return holds - } - s.load(false) - now := s.now() - s.mu.Lock() - defer s.mu.Unlock() - for host, until := range s.shared { - if now.Before(until) && until.After(holds[host]) { - if holds == nil { - holds = make(map[string]time.Time) - } - holds[host] = until - } - } - return holds -} - -// Reload reads the shared holds now rather than when the copy runs out. A -// caller that knows another instance has just failed, say on a 429, uses it -// to see that instance's hold at once. -func (s *SharedRateLimitGate) Reload() { - if s == nil || s.store == nil { - return - } - s.load(true) -} - -func (s *SharedRateLimitGate) sharedHeld(host string) error { - if s.store == nil { - return nil - } - s.load(false) - now := s.now() - s.mu.Lock() - until, ok := s.shared[host] - s.mu.Unlock() - if !ok || !now.Before(until) { - return nil - } - return ®istryclient.HeldError{Host: host, Until: until, RetryAfter: until.Sub(now)} -} - -// load refreshes the copy of the shared holds once it is older than Refresh, -// or at once when force is set. One caller reads the store while concurrent -// callers go on with the copy they have; a forced read does not wait for -// one already running, which may have started before the hold it is after. -func (s *SharedRateLimitGate) load(force bool) { - now := s.now() - s.mu.Lock() - if !force && (s.loading || now.Sub(s.loaded) < s.Refresh) { - s.mu.Unlock() - return - } - s.loading = true - s.mu.Unlock() - - holds, err := s.store.UpstreamHolds(now) - - s.mu.Lock() - defer s.mu.Unlock() - s.loading = false - // A failed read keeps the last copy and waits a full interval before - // trying again, so a database outage costs no more queries than usual. - s.loaded = now - if err != nil { - if s.logger != nil { - s.logger.Warn("reading shared upstream holds failed", "error", err) - } - if holds == nil { - return - } - } - for host, until := range s.published { - if !now.Before(until) { - delete(s.published, host) - } else if until.After(holds[host]) { - holds[host] = until - } - } - s.shared = holds -} - -// publish records a hold this instance placed, locally and in the store. The -// responses already in flight when a hold starts each report it again; only a -// later end is written. -func (s *SharedRateLimitGate) publish(host string, until time.Time) { - s.mu.Lock() - if !until.After(s.published[host]) { - s.mu.Unlock() - return - } - s.published[host] = until - if until.After(s.shared[host]) { - s.shared[host] = until - } - s.mu.Unlock() - if err := s.store.PutUpstreamHold(host, until); err != nil && s.logger != nil { - s.logger.Warn("publishing upstream hold failed", "host", host, "error", err) - } -} - -func hostKey(u *url.URL) string { - if u == nil { - return "" - } - return strings.ToLower(u.Host) -} diff --git a/internal/httpclient/shared_holds_test.go b/internal/httpclient/shared_holds_test.go deleted file mode 100644 index 31b9d79..0000000 --- a/internal/httpclient/shared_holds_test.go +++ /dev/null @@ -1,125 +0,0 @@ -package httpclient - -import ( - "errors" - "net/http" - "net/http/httptest" - "sync" - "sync/atomic" - "testing" - "time" - - registryclient "github.com/git-pkgs/registries/client" -) - -// memoryHoldStore is a HoldStore shared by gates in one test, counting reads. -type memoryHoldStore struct { - mu sync.Mutex - holds map[string]time.Time - reads atomic.Int32 -} - -func (m *memoryHoldStore) PutUpstreamHold(host string, until time.Time) error { - m.mu.Lock() - defer m.mu.Unlock() - if until.After(m.holds[host]) { - m.holds[host] = until - } - return nil -} - -func (m *memoryHoldStore) UpstreamHolds(now time.Time) (map[string]time.Time, error) { - m.reads.Add(1) - m.mu.Lock() - defer m.mu.Unlock() - out := make(map[string]time.Time) - for host, until := range m.holds { - if now.Before(until) { - out[host] = until - } - } - return out, nil -} - -func TestSharedRateLimitGateSharesHolds(t *testing.T) { - var calls atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - calls.Add(1) - w.Header().Set("Retry-After", "120") - w.WriteHeader(http.StatusTooManyRequests) - })) - defer upstream.Close() - - store := &memoryHoldStore{holds: make(map[string]time.Time)} - a := NewSharedRateLimitGate(registryclient.NewRateLimitGate(upstream.Client().Transport), store, nil) - b := NewSharedRateLimitGate(registryclient.NewRateLimitGate(upstream.Client().Transport), store, nil) - clock := time.Now() - b.now = func() time.Time { return clock } - - // b reads the store before a is rate limited, and keeps that copy for - // Refresh. - if err := b.Held(upstream.URL); err != nil { - t.Fatalf("b held before any 429: %v", err) - } - resp, err := a.RoundTrip(httptest.NewRequest(http.MethodGet, upstream.URL, nil)) - if err != nil || resp.StatusCode != http.StatusTooManyRequests { - t.Fatalf("a: %v, %v; want the 429", resp, err) - } - _ = resp.Body.Close() - if err := b.Held(upstream.URL); err != nil { - t.Errorf("b refreshed before Refresh ran out: %v", err) - } - - clock = clock.Add(DefaultHoldRefresh) - _, err = b.RoundTrip(httptest.NewRequest(http.MethodGet, upstream.URL, nil)) - var held *registryclient.HeldError - if !errors.As(err, &held) || held.RetryAfter <= 100*time.Second { - t.Fatalf("b after refresh: %v, want a hold of about two minutes", err) - } - if calls.Load() != 1 { - t.Errorf("upstream requests = %d, want 1", calls.Load()) - } - if holds := b.Holds(); len(holds) != 1 { - t.Errorf("b.Holds() = %v, want a's hold", holds) - } - // One read by a before its request, two by b: the first and the one - // after Refresh. - if n := store.reads.Load(); n != 3 { - t.Errorf("store reads = %d, want 3", n) - } - - // Once the hold ends, b sends again. - clock = clock.Add(2 * time.Minute) - if err := b.Held(upstream.URL); err != nil { - t.Errorf("b after the hold ended: %v", err) - } -} - -func TestSharedRateLimitGateReload(t *testing.T) { - store := &memoryHoldStore{holds: make(map[string]time.Time)} - g := NewSharedRateLimitGate(registryclient.NewRateLimitGate(nil), store, nil) - g.Refresh = time.Hour - if err := g.Held("https://registry.example/x"); err != nil { - t.Fatal(err) - } - _ = store.PutUpstreamHold("registry.example", time.Now().Add(time.Minute)) - if err := g.Held("https://registry.example/x"); err != nil { - t.Fatalf("held before reload: %v", err) - } - g.Reload() - if !errors.Is(g.Held("https://registry.example/x"), registryclient.ErrUpstreamHeld) { - t.Error("not held after reload") - } -} - -func TestSharedRateLimitGateWithoutStore(t *testing.T) { - g := NewSharedRateLimitGate(registryclient.NewRateLimitGate(nil), nil, nil) - if err := g.Held("https://registry.example/x"); err != nil { - t.Fatal(err) - } - g.Reload() - var nilGate *SharedRateLimitGate - if nilGate.Held("https://registry.example/x") != nil || nilGate.Holds() != nil { - t.Error("nil gate holds") - } -} diff --git a/internal/httpclient/token_scope_test.go b/internal/httpclient/token_scope_test.go deleted file mode 100644 index 563d939..0000000 --- a/internal/httpclient/token_scope_test.go +++ /dev/null @@ -1,241 +0,0 @@ -package httpclient - -import ( - "io" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "sync/atomic" - "testing" -) - -// tokenService issues "secret-token" only to requests carrying the configured -// credentials, and "anonymous-token" otherwise. -func tokenService(t *testing.T, issued *atomic.Int32) *httptest.Server { - t.Helper() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - issued.Add(1) - token := "anonymous-token" - if r.Header.Get("Authorization") == "Basic Y29ycDpwYXNz" { - token = "secret-token" - } - _, _ = io.WriteString(w, `{"token":"`+token+`","expires_in":3600}`) - })) - t.Cleanup(srv.Close) - return srv -} - -// challengingRegistry answers every request without a bearer token with a -// challenge naming realm, and records the Authorization headers it receives. -func challengingRegistry(t *testing.T, realm string, received *[]string) *httptest.Server { - t.Helper() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - auth := r.Header.Get("Authorization") - *received = append(*received, auth) - if !strings.HasPrefix(auth, "Bearer ") { - w.Header().Set("WWW-Authenticate", `Bearer realm="`+realm+`/token",service="registry.corp",scope="repository:corp/private:pull"`) - http.Error(w, "authentication required", http.StatusUnauthorized) - return - } - _, _ = io.WriteString(w, "blob") - })) - t.Cleanup(srv.Close) - return srv -} - -func credentialsFor(prefix string) AuthFunc { - return func(rawURL string) (string, string) { - if strings.HasPrefix(rawURL, prefix) { - return "Authorization", "Basic Y29ycDpwYXNz" - } - return "", "" - } -} - -func get(t *testing.T, client *http.Client, rawURL string) int { - t.Helper() - resp, err := client.Get(rawURL) - if err != nil { - t.Fatalf("GET %s: %v", rawURL, err) - } - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - return resp.StatusCode -} - -// TestTransportDoesNotSendCredentialedTokenToAnotherSite is a regression test -// for a credential leak. Any host the proxy fetches from, such as one named in -// a Composer dist URL or a Helm index, could answer with a challenge naming a -// token service the proxy has credentials for. The proxy fetched a token with -// those credentials and sent it to that host. -func TestTransportDoesNotSendCredentialedTokenToAnotherSite(t *testing.T) { - var issued atomic.Int32 - realm := tokenService(t, &issued) - var received []string - evil := challengingRegistry(t, realm.URL, &received) - - client := &http.Client{Transport: NewTransport(http.DefaultTransport, credentialsFor(realm.URL))} - if status := get(t, client, evil.URL+"/v2/corp/private/blobs/sha256:abc"); status != http.StatusUnauthorized { - t.Errorf("status = %d, want the challenge passed back as 401", status) - } - for _, auth := range received { - if auth != "" { - t.Errorf("the challenging host received Authorization %q", auth) - } - } - if n := issued.Load(); n != 0 { - t.Errorf("token service was asked %d times on the challenging host's behalf, want 0", n) - } -} - -// TestTransportDoesNotSendCachedCredentialedTokenToAnotherSite checks the -// same for a token already cached from a legitimate pull: a challenge that -// names the same realm, service and scope must not be handed it. -func TestTransportDoesNotSendCachedCredentialedTokenToAnotherSite(t *testing.T) { - var issued atomic.Int32 - realm := tokenService(t, &issued) - var legitimate, leaked []string - registry := challengingRegistry(t, realm.URL, &legitimate) - evil := challengingRegistry(t, realm.URL, &leaked) - - // The registry itself has configured credentials, so it is trusted with - // tokens from its realm. - auth := func(rawURL string) (string, string) { - if strings.HasPrefix(rawURL, realm.URL) || strings.HasPrefix(rawURL, registry.URL) { - return "Authorization", "Basic Y29ycDpwYXNz" - } - return "", "" - } - client := &http.Client{Transport: NewTransport(http.DefaultTransport, auth)} - if status := get(t, client, registry.URL+"/v2/corp/private/blobs/sha256:abc"); status != http.StatusOK { - t.Fatalf("legitimate pull: status = %d, want 200", status) - } - if got := legitimate[len(legitimate)-1]; got != "Bearer secret-token" { - t.Fatalf("legitimate pull sent %q, want the credentialed token", got) - } - - get(t, client, evil.URL+"/v2/corp/private/blobs/sha256:abc") - for _, got := range leaked { - if strings.HasPrefix(got, "Bearer ") { - t.Errorf("the challenging host received %q", got) - } - } -} - -// TestTransportFollowsAnonymousTokenServiceOnAnotherHost keeps public pulls -// working where the token service is on another host and no credentials are -// configured for it: the token is anonymous, so any registry may have it. -func TestTransportFollowsAnonymousTokenServiceOnAnotherHost(t *testing.T) { - var issued atomic.Int32 - realm := tokenService(t, &issued) - var received []string - registry := challengingRegistry(t, realm.URL, &received) - - client := &http.Client{Transport: NewTransport(http.DefaultTransport, credentialsFor("https://elsewhere.example"))} - if status := get(t, client, registry.URL+"/v2/library/alpine/blobs/sha256:abc"); status != http.StatusOK { - t.Errorf("status = %d, want 200", status) - } - if got := received[len(received)-1]; got != "Bearer anonymous-token" { - t.Errorf("registry received %q, want the anonymous token", got) - } -} - -// TestTransportDoesNotSendCredentialedTokenToHostWithOtherCredentials checks -// that a registry with credentials of its own, but not the realm's, is not -// handed a token minted from the realm's credentials. -func TestTransportDoesNotSendCredentialedTokenToHostWithOtherCredentials(t *testing.T) { - var issued atomic.Int32 - realm := tokenService(t, &issued) - var received []string - vendor := challengingRegistry(t, realm.URL, &received) - - auth := func(rawURL string) (string, string) { - switch { - case strings.HasPrefix(rawURL, realm.URL): - return "Authorization", "Basic Y29ycDpwYXNz" - case strings.HasPrefix(rawURL, vendor.URL): - return "Authorization", "Basic dmVuZG9yOmtleQ==" - } - return "", "" - } - client := &http.Client{Transport: NewTransport(http.DefaultTransport, auth)} - if status := get(t, client, vendor.URL+"/v2/corp/private/blobs/sha256:abc"); status != http.StatusUnauthorized { - t.Errorf("status = %d, want the challenge passed back as 401", status) - } - for _, got := range received { - if got != "Basic dmVuZG9yOmtleQ==" { - t.Errorf("the vendor registry received Authorization %q, want only its own credentials", got) - } - } - if n := issued.Load(); n != 0 { - t.Errorf("token service was asked %d times on the vendor registry's behalf, want 0", n) - } -} - -// TestTransportRequestsPullScopeForRequestedRepository checks that the token -// is requested for pull access to the repository in the request path, not -// whatever scope the challenge names. -func TestTransportRequestsPullScopeForRequestedRepository(t *testing.T) { - var scopes [][]string - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/token" { - scopes = append(scopes, r.URL.Query()["scope"]) - _, _ = io.WriteString(w, `{"token":"t","expires_in":3600}`) - return - } - if r.Header.Get("Authorization") != "Bearer t" { - w.Header().Set("WWW-Authenticate", `Bearer realm="`+srv.URL+`/token",service="registry.corp",scope="repository:corp/other:pull,push"`) - http.Error(w, "authentication required", http.StatusUnauthorized) - return - } - _, _ = io.WriteString(w, "blob") - })) - t.Cleanup(srv.Close) - - client := &http.Client{Transport: NewTransport(http.DefaultTransport, credentialsFor(srv.URL))} - if status := get(t, client, srv.URL+"/v2/corp/app/manifests/latest"); status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if len(scopes) != 1 || len(scopes[0]) != 1 || scopes[0][0] != "repository:corp/app:pull" { - t.Errorf("token requested with scopes %q, want [[repository:corp/app:pull]]", scopes) - } -} - -func TestSameTokenService(t *testing.T) { - for _, tt := range []struct { - registry, realm string - want bool - }{ - {"https://registry-1.docker.io", "https://auth.docker.io/token", true}, - {"https://index.docker.io", "https://auth.docker.io/token", true}, - {"https://registry.gitlab.com", "https://gitlab.com/jwt/auth", true}, - {"https://ghcr.io", "https://ghcr.io/token", true}, - {"https://harbor.corp.example", "https://harbor.corp.example/service/token", true}, - {"https://registry.corp.example", "https://auth.corp.example", false}, - {"https://evil.example", "https://auth.docker.io/token", false}, - {"https://docker.io.evil.example", "https://auth.docker.io/token", false}, - {"https://evil.jfrog.io", "https://myco.jfrog.io/artifactory/api/docker/token", false}, - {"https://evil.azurecr.io", "https://myco.azurecr.io/oauth2/token", false}, - {"https://gitlab.com", "https://registry.gitlab.com", false}, - {"http://registry-1.docker.io", "https://auth.docker.io/token", false}, - {"https://registry-1.docker.io", "https://auth.docker.io:8443/token", false}, - {"http://127.0.0.1:5000", "http://127.0.0.1:5000/token", true}, - {"http://127.0.0.1:5000", "http://127.0.0.1:5001/token", false}, - {"http://localhost:5000", "https://localhost:5000/token", false}, - } { - if got := sameTokenService(mustParse(t, tt.registry), mustParse(t, tt.realm)); got != tt.want { - t.Errorf("sameTokenService(%s, %s) = %v, want %v", tt.registry, tt.realm, got, tt.want) - } - } -} - -func mustParse(t *testing.T, rawURL string) *url.URL { - t.Helper() - u, err := url.Parse(rawURL) - if err != nil { - t.Fatal(err) - } - return u -} diff --git a/internal/httpclient/transport.go b/internal/httpclient/transport.go deleted file mode 100644 index a007c8d..0000000 --- a/internal/httpclient/transport.go +++ /dev/null @@ -1,597 +0,0 @@ -// Package httpclient provides authentication-aware HTTP transports for upstream requests. -package httpclient - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net" - "net/http" - "net/url" - "strings" - "sync" - "time" -) - -const ( - defaultTokenLifetime = 60 * time.Second - tokenExpirySkew = 5 * time.Second - maxTokenResponseSize = 1 << 20 - shortTokenSkewDivisor = 10 - tokenMaxRetries = 3 - tokenRetryBaseDelay = 500 * time.Millisecond - - // maxTokenLifetime caps how long a token is reused, whatever lifetime - // the token service claims. - maxTokenLifetime = time.Hour - // maxCachedTokens and maxCachedChallenges bound the token and - // challenge caches, which grow with every repository and registry - // host the proxy pulls from. - maxCachedTokens = 1024 - maxCachedChallenges = 256 -) - -// AuthFunc returns a configured authentication header for a URL. -type AuthFunc func(url string) (headerName, headerValue string) - -// Transport adds configured authentication and follows OCI Bearer challenges. -type Transport struct { - base http.RoundTripper - authForURL AuthFunc - retryWait func(context.Context, time.Duration) error - - mu sync.Mutex - tokens map[string]cachedToken - challenges map[string]bearerChallenge // by registry origin, without scopes -} - -type cachedToken struct { - value string - expiresAt time.Time -} - -type bearerChallenge struct { - realm string - service string - scopes []string -} - -type tokenResponse struct { - Token string `json:"token"` - AccessToken string `json:"access_token"` - ExpiresIn int64 `json:"expires_in"` - IssuedAt string `json:"issued_at"` -} - -// NewTransport creates an authentication-aware transport around base. -func NewTransport(base http.RoundTripper, authForURL AuthFunc) *Transport { - if base == nil { - base = http.DefaultTransport - } - return &Transport{ - base: base, - authForURL: authForURL, - retryWait: waitForRetry, - tokens: make(map[string]cachedToken), - challenges: make(map[string]bearerChallenge), - } -} - -// RoundTrip implements http.RoundTripper. -func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { - hasExplicitAuthorization := req.Header.Get("Authorization") != "" - outbound := cloneRequest(req) - t.applyAuthentication(outbound, hasExplicitAuthorization) - - resp, err := t.base.RoundTrip(outbound) - if err != nil || resp.StatusCode != http.StatusUnauthorized { - return resp, err - } - if hasExplicitAuthorization { - return resp, nil - } - repository := registryRepository(req.URL) - if repository == "" { - return resp, nil - } - - challenge, ok := parseBearerChallenge(resp.Header.Values("WWW-Authenticate")) - // Ask only for pull access to the repository being fetched, whatever - // scope the challenge names: the proxy only reads. - challenge.scopes = []string{pullScope(repository)} - if !ok || !canReplay(req) || !t.trustedWithToken(req.URL, challenge) { - return resp, nil - } - - drainAndClose(resp.Body) - token, err := t.token(req.Context(), challenge) - if err != nil { - return nil, fmt.Errorf("registry authentication: %w", err) - } - t.rememberChallenge(req.URL, challenge) - - retry, err := cloneRequestForRetry(req) - if err != nil { - return nil, err - } - t.applyConfiguredAuthentication(retry) - retry.Header.Set("Authorization", "Bearer "+token) - return t.base.RoundTrip(retry) -} - -// trustedWithToken reports whether a token from challenge's realm may be -// fetched for, and sent to, the registry at requestURL. The challenge comes -// from that registry, which can be any host the proxy fetches from, such as -// one named in a Composer dist URL or a Helm index. Credentials configured for -// the realm would otherwise be turned into a token handed to that host. A -// token from a realm with no configured credentials is anonymous, and any -// registry may have it. A credentialed token only goes to the realm's own -// host, to a registry configured with the same credentials as the realm, or -// to a registry in knownTokenServices. -func (t *Transport) trustedWithToken(requestURL *url.URL, challenge bearerChallenge) bool { - if t.authForURL == nil { - return true - } - realmName, realmValue := t.authForURL(challenge.realm) - if realmName == "" || realmValue == "" { - return true - } - if name, value := t.authForURL(requestURL.String()); name == realmName && value == realmValue { - return true - } - realm, err := url.Parse(challenge.realm) - return err == nil && sameTokenService(requestURL, realm) -} - -// knownTokenServices maps registry hosts to the token service host they -// use where the two differ and the registry cannot be configured with the -// token service's credentials under the same upstream.auth key. -var knownTokenServices = map[string]string{ - "registry-1.docker.io": "auth.docker.io", - "index.docker.io": "auth.docker.io", - "registry.gitlab.com": "gitlab.com", -} - -// sameTokenService reports whether a token service at realm belongs to the -// registry at registry: both on the same scheme, host and port, or a known -// pair of registry and token service hosts. -func sameTokenService(registry, realm *url.URL) bool { - if strings.EqualFold(registry.Scheme, realm.Scheme) && strings.EqualFold(registry.Host, realm.Host) { - return true - } - if registry.Scheme != "https" || realm.Scheme != "https" || registry.Port() != "" || realm.Port() != "" { - return false - } - want, ok := knownTokenServices[strings.ToLower(registry.Hostname())] - return ok && strings.EqualFold(realm.Hostname(), want) -} - -func (t *Transport) applyAuthentication(req *http.Request, hasExplicitAuthorization bool) { - t.applyConfiguredAuthentication(req) - if hasExplicitAuthorization { - return - } - if token := t.cachedTokenForRequest(req.URL); token != "" { - req.Header.Set("Authorization", "Bearer "+token) - } -} - -func (t *Transport) applyConfiguredAuthentication(req *http.Request) { - if t.authForURL == nil { - return - } - name, value := t.authForURL(req.URL.String()) - if name != "" && value != "" && req.Header.Get(name) == "" { - req.Header.Set(name, value) - } -} - -func (t *Transport) token(ctx context.Context, challenge bearerChallenge) (string, error) { - key := challenge.key() - if token := t.cachedToken(key); token != "" { - return token, nil - } - - token, expiresAt, err := t.fetchToken(ctx, challenge) - if err != nil { - return "", err - } - - t.cacheToken(key, cachedToken{value: token, expiresAt: expiresAt}) - return token, nil -} - -func (t *Transport) cacheToken(key string, token cachedToken) { - now := time.Now() - t.mu.Lock() - defer t.mu.Unlock() - - for cachedKey, cached := range t.tokens { - if !now.Before(cached.expiresAt) { - delete(t.tokens, cachedKey) - } - } - if _, ok := t.tokens[key]; !ok && len(t.tokens) >= maxCachedTokens { - // Drop the token that expires soonest. - var oldestKey string - var oldest time.Time - for cachedKey, cached := range t.tokens { - if oldestKey == "" || cached.expiresAt.Before(oldest) { - oldestKey, oldest = cachedKey, cached.expiresAt - } - } - delete(t.tokens, oldestKey) - } - t.tokens[key] = token -} - -func (t *Transport) fetchToken(ctx context.Context, challenge bearerChallenge) (string, time.Time, error) { - tokenURL, err := url.Parse(challenge.realm) - if err != nil || !tokenURL.IsAbs() || (tokenURL.Scheme != "https" && tokenURL.Scheme != "http") { - return "", time.Time{}, fmt.Errorf("invalid token realm %q", challenge.realm) - } - - query := tokenURL.Query() - if challenge.service != "" { - query.Set("service", challenge.service) - } - for _, scope := range challenge.scopes { - query.Add("scope", scope) - } - query.Set("client_id", "git-pkgs-proxy") - tokenURL.RawQuery = query.Encode() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL.String(), nil) - if err != nil { - return "", time.Time{}, err - } - - client := &http.Client{Transport: configuredTransport{parent: t}} - for attempt := 0; attempt <= tokenMaxRetries; attempt++ { - resp, err := client.Do(req.Clone(ctx)) - if err != nil { - requestErr := fmt.Errorf("requesting token: %w", err) - if !shouldRetryTokenRequest(ctx, err) || attempt == tokenMaxRetries { - return "", time.Time{}, requestErr - } - if err := t.waitForTokenRetry(ctx, attempt); err != nil { - return "", time.Time{}, err - } - continue - } - - if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices { - return decodeTokenResponse(resp) - } - - responseErr := tokenResponseError(resp) - if !shouldRetryTokenStatus(resp.StatusCode) || attempt == tokenMaxRetries { - return "", time.Time{}, responseErr - } - if err := t.waitForTokenRetry(ctx, attempt); err != nil { - return "", time.Time{}, err - } - } - - return "", time.Time{}, errors.New("token request retries exhausted") -} - -func decodeTokenResponse(resp *http.Response) (string, time.Time, error) { - defer func() { _ = resp.Body.Close() }() - - var payload tokenResponse - if err := json.NewDecoder(io.LimitReader(resp.Body, maxTokenResponseSize)).Decode(&payload); err != nil { - return "", time.Time{}, fmt.Errorf("decoding token response: %w", err) - } - token := payload.Token - if token == "" { - token = payload.AccessToken - } - if token == "" { - return "", time.Time{}, fmt.Errorf("token response did not contain a token") - } - - issuedAt := time.Now() - if payload.IssuedAt != "" { - if parsed, parseErr := time.Parse(time.RFC3339, payload.IssuedAt); parseErr == nil { - issuedAt = parsed - } - } - lifetime := time.Duration(payload.ExpiresIn) * time.Second - if lifetime <= 0 { - lifetime = defaultTokenLifetime - } - lifetime = min(lifetime, maxTokenLifetime) - expiresAt := issuedAt.Add(lifetime).Add(-expirySkew(lifetime)) - return token, expiresAt, nil -} - -func tokenResponseError(resp *http.Response) error { - defer func() { _ = resp.Body.Close() }() - body, _ := io.ReadAll(io.LimitReader(resp.Body, maxTokenResponseSize)) - return fmt.Errorf("token service returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) -} - -func shouldRetryTokenRequest(ctx context.Context, err error) bool { - if ctx.Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return false - } - - var networkErr net.Error - if !errors.As(err, &networkErr) { - return false - } - - var dnsErr *net.DNSError - if errors.As(err, &dnsErr) { - return dnsErr.IsTemporary || dnsErr.IsTimeout - } - return networkErr.Timeout() -} - -func shouldRetryTokenStatus(status int) bool { - return status == http.StatusTooManyRequests || status >= http.StatusInternalServerError -} - -func (t *Transport) waitForTokenRetry(ctx context.Context, attempt int) error { - delay := tokenRetryBaseDelay << attempt - if t.retryWait != nil { - return t.retryWait(ctx, delay) - } - return waitForRetry(ctx, delay) -} - -func waitForRetry(ctx context.Context, delay time.Duration) error { - timer := time.NewTimer(delay) - defer timer.Stop() - - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - return nil - } -} - -type configuredTransport struct { - parent *Transport -} - -func (t configuredTransport) RoundTrip(req *http.Request) (*http.Response, error) { - outbound := cloneRequest(req) - t.parent.applyConfiguredAuthentication(outbound) - return t.parent.base.RoundTrip(outbound) -} - -func (t *Transport) cachedTokenForRequest(requestURL *url.URL) string { - repository := registryRepository(requestURL) - if repository == "" { - return "" - } - - t.mu.Lock() - challenge, ok := t.challenges[registryOrigin(requestURL)] - t.mu.Unlock() - if !ok { - return "" - } - challenge.scopes = []string{pullScope(repository)} - return t.cachedToken(challenge.key()) -} - -func (t *Transport) cachedToken(key string) string { - now := time.Now() - t.mu.Lock() - defer t.mu.Unlock() - - token, ok := t.tokens[key] - if !ok { - return "" - } - if !now.Before(token.expiresAt) { - delete(t.tokens, key) - return "" - } - return token.value -} - -// rememberChallenge records the token service a registry named, so later -// requests to any of its repositories can send a cached token up front. -func (t *Transport) rememberChallenge(requestURL *url.URL, challenge bearerChallenge) { - origin := registryOrigin(requestURL) - challenge.scopes = nil - t.mu.Lock() - defer t.mu.Unlock() - if _, ok := t.challenges[origin]; !ok && len(t.challenges) >= maxCachedChallenges { - for evict := range t.challenges { - delete(t.challenges, evict) - break - } - } - t.challenges[origin] = challenge -} - -func (c bearerChallenge) key() string { - return c.realm + "\x00" + c.service + "\x00" + strings.Join(c.scopes, "\x00") -} - -func registryOrigin(u *url.URL) string { - return strings.ToLower(u.Scheme + "://" + u.Host) -} - -func pullScope(repository string) string { - return "repository:" + repository + ":pull" -} - -// registryRepository returns the repository name from an OCI distribution -// API path, such as library/alpine from /v2/library/alpine/manifests/latest, -// or "" if the path is not one. -func registryRepository(u *url.URL) string { - const registryPrefix = "/v2/" - if u == nil || !strings.HasPrefix(u.Path, registryPrefix) { - return "" - } - rest := strings.TrimPrefix(u.Path, registryPrefix) - end := len(rest) - for _, marker := range []string{"/blobs/", "/manifests/", "/tags/", "/referrers/"} { - if index := strings.Index(rest, marker); index >= 0 && index < end { - end = index - } - } - if end == len(rest) || end == 0 { - return "" - } - return rest[:end] -} - -func parseBearerChallenge(values []string) (bearerChallenge, bool) { - for _, value := range values { - params, ok := bearerParameters(value) - if !ok || params["realm"] == "" { - continue - } - challenge := bearerChallenge{ - realm: params["realm"], - service: params["service"], - } - if scope := params["scope"]; scope != "" { - challenge.scopes = append(challenge.scopes, scope) - } - return challenge, true - } - return bearerChallenge{}, false -} - -func bearerParameters(value string) (map[string]string, bool) { - start := findAuthScheme(value, "Bearer") - if start < 0 { - return nil, false - } - rest := value[start+len("Bearer"):] - params := make(map[string]string) - for { - rest = strings.TrimLeft(rest, " \t,") - if rest == "" { - break - } - - keyEnd := strings.IndexAny(rest, "= \t,") - if keyEnd <= 0 { - break - } - key := strings.ToLower(rest[:keyEnd]) - rest = strings.TrimLeft(rest[keyEnd:], " \t") - if rest == "" || rest[0] != '=' { - break - } - rest = strings.TrimLeft(rest[1:], " \t") - - parsed, remaining, ok := parseAuthValue(rest) - if !ok { - return nil, false - } - params[key] = parsed - rest = remaining - } - return params, true -} - -func findAuthScheme(value, scheme string) int { - inQuote := false - escaped := false - for index := 0; index+len(scheme) <= len(value); index++ { - char := value[index] - if escaped { - escaped = false - continue - } - if char == '\\' && inQuote { - escaped = true - continue - } - if char == '"' { - inQuote = !inQuote - continue - } - if inQuote || !strings.EqualFold(value[index:index+len(scheme)], scheme) { - continue - } - beforeOK := index == 0 || value[index-1] == ',' || value[index-1] == ' ' || value[index-1] == '\t' - after := index + len(scheme) - afterOK := after < len(value) && (value[after] == ' ' || value[after] == '\t') - if beforeOK && afterOK { - return index - } - } - return -1 -} - -func parseAuthValue(value string) (parsed, remaining string, ok bool) { - if value == "" { - return "", "", false - } - if value[0] != '"' { - end := strings.IndexAny(value, " \t,") - if end < 0 { - return value, "", true - } - return value[:end], value[end:], end > 0 - } - - var builder strings.Builder - escaped := false - for index := 1; index < len(value); index++ { - char := value[index] - if escaped { - builder.WriteByte(char) - escaped = false - continue - } - if char == '\\' { - escaped = true - continue - } - if char == '"' { - return builder.String(), value[index+1:], true - } - builder.WriteByte(char) - } - return "", "", false -} - -func cloneRequest(req *http.Request) *http.Request { - clone := req.Clone(req.Context()) - clone.Header = req.Header.Clone() - return clone -} - -func canReplay(req *http.Request) bool { - return req.Body == nil || req.GetBody != nil -} - -func cloneRequestForRetry(req *http.Request) (*http.Request, error) { - clone := cloneRequest(req) - if req.Body == nil { - return clone, nil - } - body, err := req.GetBody() - if err != nil { - return nil, fmt.Errorf("replaying authenticated request: %w", err) - } - clone.Body = body - return clone, nil -} - -func expirySkew(lifetime time.Duration) time.Duration { - if lifetime < tokenExpirySkew*2 { - return lifetime / shortTokenSkewDivisor - } - return tokenExpirySkew -} - -func drainAndClose(body io.ReadCloser) { - _, _ = io.Copy(io.Discard, io.LimitReader(body, maxTokenResponseSize)) - _ = body.Close() -} diff --git a/internal/httpclient/transport_test.go b/internal/httpclient/transport_test.go deleted file mode 100644 index 46a31b6..0000000 --- a/internal/httpclient/transport_test.go +++ /dev/null @@ -1,463 +0,0 @@ -package httpclient - -import ( - "context" - "errors" - "fmt" - "io" - "net" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" -) - -type roundTripperFunc func(*http.Request) (*http.Response, error) - -func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { - return fn(req) -} - -func TestTransportFollowsBearerChallengeAndCachesToken(t *testing.T) { - var registryRequests int - var tokenRequests int - var server *httptest.Server - - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/token": - tokenRequests++ - if got := r.URL.Query().Get("service"); got != "registry.test" { - t.Errorf("service = %q, want %q", got, "registry.test") - } - if got := r.URL.Query().Get("scope"); got != "repository:library/test:pull" { - t.Errorf("scope = %q, want %q", got, "repository:library/test:pull") - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"token":"registry-token","expires_in":3600}`) - case "/v2/library/test/blobs/sha256:first", "/v2/library/test/blobs/sha256:second": - registryRequests++ - if r.Header.Get("Authorization") != "Bearer registry-token" { - w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token",service="registry.test",scope="repository:library/test:pull"`) - http.Error(w, "authentication required", http.StatusUnauthorized) - return - } - _, _ = io.WriteString(w, "blob") - default: - http.NotFound(w, r) - } - })) - defer server.Close() - - client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)} - for _, digest := range []string{"sha256:first", "sha256:second"} { - resp, err := client.Get(server.URL + "/v2/library/test/blobs/" + digest) - if err != nil { - t.Fatalf("GET %s: %v", digest, err) - } - body, readErr := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if readErr != nil { - t.Fatalf("read %s response: %v", digest, readErr) - } - if resp.StatusCode != http.StatusOK { - t.Fatalf("GET %s status = %d, want %d", digest, resp.StatusCode, http.StatusOK) - } - if string(body) != "blob" { - t.Errorf("GET %s body = %q, want %q", digest, body, "blob") - } - } - - if tokenRequests != 1 { - t.Errorf("token requests = %d, want 1", tokenRequests) - } - if registryRequests != 3 { - t.Errorf("registry requests = %d, want 3", registryRequests) - } -} - -func TestTransportRetriesTemporaryTokenLookupFailures(t *testing.T) { - var registryRequests int - var tokenRequests int - var tokenLookupFailures int - var server *httptest.Server - - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/token": - tokenRequests++ - _, _ = io.WriteString(w, `{"token":"registry-token"}`) - case "/v2/library/test/blobs/sha256:test": - registryRequests++ - if r.Header.Get("Authorization") != "Bearer registry-token" { - w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token",service="registry.test",scope="repository:library/test:pull"`) - http.Error(w, "authentication required", http.StatusUnauthorized) - return - } - _, _ = io.WriteString(w, "blob") - default: - http.NotFound(w, r) - } - })) - defer server.Close() - - base := roundTripperFunc(func(req *http.Request) (*http.Response, error) { - if req.URL.Path == "/token" && tokenLookupFailures < 2 { - tokenLookupFailures++ - return nil, &net.DNSError{Err: "server misbehaving", IsTemporary: true} - } - return http.DefaultTransport.RoundTrip(req) - }) - transport := NewTransport(base, nil) - transport.retryWait = func(context.Context, time.Duration) error { return nil } - client := &http.Client{Transport: transport} - - resp, err := client.Get(server.URL + "/v2/library/test/blobs/sha256:test") - if err != nil { - t.Fatalf("GET blob: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK) - } - if tokenLookupFailures != 2 { - t.Errorf("token lookup failures = %d, want 2", tokenLookupFailures) - } - if tokenRequests != 1 { - t.Errorf("token requests = %d, want 1", tokenRequests) - } - if registryRequests != 2 { - t.Errorf("registry requests = %d, want 2", registryRequests) - } -} - -func TestTransportDoesNotRetryPermanentTokenLookupFailures(t *testing.T) { - var tokenRequests int - base := roundTripperFunc(func(*http.Request) (*http.Response, error) { - tokenRequests++ - return nil, &net.DNSError{Err: "no such host"} - }) - transport := NewTransport(base, nil) - transport.retryWait = func(context.Context, time.Duration) error { return nil } - - _, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: "https://auth.example.test/token"}) - if err == nil { - t.Fatal("fetchToken succeeded, want error") - } - if tokenRequests != 1 { - t.Errorf("token requests = %d, want 1", tokenRequests) - } -} - -func TestTransportDoesNotRetryPermanentTokenFailures(t *testing.T) { - var tokenRequests int - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - tokenRequests++ - http.Error(w, "invalid credentials", http.StatusUnauthorized) - })) - defer server.Close() - - transport := NewTransport(http.DefaultTransport, nil) - _, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: server.URL + "/token"}) - if err == nil { - t.Fatal("fetchToken succeeded, want error") - } - if tokenRequests != 1 { - t.Errorf("token requests = %d, want 1", tokenRequests) - } -} - -func TestTransportRetriesTokenServiceFailures(t *testing.T) { - for _, status := range []int{http.StatusTooManyRequests, http.StatusServiceUnavailable} { - t.Run(http.StatusText(status), func(t *testing.T) { - var tokenRequests int - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - tokenRequests++ - http.Error(w, "temporary token service failure", status) - })) - defer server.Close() - - var delays []time.Duration - transport := NewTransport(http.DefaultTransport, nil) - transport.retryWait = func(_ context.Context, delay time.Duration) error { - delays = append(delays, delay) - return nil - } - - _, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: server.URL + "/token"}) - if err == nil { - t.Fatal("fetchToken succeeded, want error") - } - if tokenRequests != tokenMaxRetries+1 { - t.Errorf("token requests = %d, want %d", tokenRequests, tokenMaxRetries+1) - } - wantDelays := []time.Duration{500 * time.Millisecond, time.Second, 2 * time.Second} - if len(delays) != len(wantDelays) { - t.Fatalf("retry delays = %v, want %v", delays, wantDelays) - } - for index, want := range wantDelays { - if delays[index] != want { - t.Errorf("retry delay %d = %s, want %s", index, delays[index], want) - } - } - }) - } -} - -func TestTransportStopsTokenRetriesWhenWaitingIsCancelled(t *testing.T) { - var tokenRequests int - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - tokenRequests++ - http.Error(w, "temporary token service failure", http.StatusServiceUnavailable) - })) - defer server.Close() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - transport := NewTransport(http.DefaultTransport, nil) - transport.retryWait = func(ctx context.Context, _ time.Duration) error { - cancel() - <-ctx.Done() - return ctx.Err() - } - - _, _, err := transport.fetchToken(ctx, bearerChallenge{realm: server.URL + "/token"}) - if !errors.Is(err, context.Canceled) { - t.Errorf("fetchToken error = %v, want context canceled", err) - } - if tokenRequests != 1 { - t.Errorf("token requests = %d, want 1", tokenRequests) - } -} - -func TestTransportAddsConfiguredAuthentication(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.Header.Get("X-Registry-Token"); got != "configured-token" { - t.Errorf("X-Registry-Token = %q, want %q", got, "configured-token") - } - w.WriteHeader(http.StatusNoContent) - })) - defer server.Close() - - authForURL := func(url string) (string, string) { - if strings.HasPrefix(url, server.URL) { - return "X-Registry-Token", "configured-token" - } - return "", "" - } - client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)} - - resp, err := client.Get(server.URL + "/metadata") - if err != nil { - t.Fatalf("GET metadata: %v", err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusNoContent { - t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent) - } -} - -func TestTransportPreservesExplicitAuthentication(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" { - t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token") - } - w.WriteHeader(http.StatusNoContent) - })) - defer server.Close() - - authForURL := func(string) (string, string) { - return "Authorization", "Bearer configured-token" - } - client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)} - req, err := http.NewRequest(http.MethodGet, server.URL+"/artifact", nil) - if err != nil { - t.Fatal(err) - } - req.Header.Set("Authorization", "Bearer explicit-token") - - resp, err := client.Do(req) - if err != nil { - t.Fatalf("GET artifact: %v", err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusNoContent { - t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent) - } -} - -func TestTransportDoesNotReplaceExplicitAuthenticationAfterBearerChallenge(t *testing.T) { - var registryRequests int - var tokenRequests int - var server *httptest.Server - - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/token": - tokenRequests++ - _, _ = io.WriteString(w, `{"token":"registry-token"}`) - case "/v2/library/test/blobs/sha256:test": - registryRequests++ - if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" { - t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token") - } - w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`) - http.Error(w, "authentication required", http.StatusUnauthorized) - default: - http.NotFound(w, r) - } - })) - defer server.Close() - - client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)} - req, err := http.NewRequest(http.MethodGet, server.URL+"/v2/library/test/blobs/sha256:test", nil) - if err != nil { - t.Fatal(err) - } - req.Header.Set("Authorization", "Bearer explicit-token") - - resp, err := client.Do(req) - if err != nil { - t.Fatalf("GET blob: %v", err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusUnauthorized { - t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized) - } - if registryRequests != 1 { - t.Errorf("registry requests = %d, want 1", registryRequests) - } - if tokenRequests != 0 { - t.Errorf("token requests = %d, want 0", tokenRequests) - } -} - -func TestTransportDoesNotForwardConfiguredAuthenticationOnTokenRedirect(t *testing.T) { - destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.Header.Get("X-Registry-Token"); got != "" { - t.Errorf("redirected X-Registry-Token = %q, want empty", got) - } - _, _ = io.WriteString(w, `{"token":"registry-token"}`) - })) - defer destination.Close() - - source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.Header.Get("X-Registry-Token"); got != "configured-token" { - t.Errorf("source X-Registry-Token = %q, want %q", got, "configured-token") - } - http.Redirect(w, r, destination.URL+"/token", http.StatusFound) - })) - defer source.Close() - - authForURL := func(rawURL string) (string, string) { - if strings.HasPrefix(rawURL, source.URL) { - return "X-Registry-Token", "configured-token" - } - return "", "" - } - transport := NewTransport(http.DefaultTransport, authForURL) - token, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: source.URL + "/token"}) - if err != nil { - t.Fatalf("fetchToken: %v", err) - } - if token != "registry-token" { - t.Errorf("token = %q, want %q", token, "registry-token") - } -} - -func TestTransportPrunesExpiredTokens(t *testing.T) { - transport := NewTransport(http.DefaultTransport, nil) - transport.tokens["expired-unused"] = cachedToken{ - value: "expired-token", - expiresAt: time.Now().Add(-time.Minute), - } - transport.cacheToken("current", cachedToken{ - value: "current-token", - expiresAt: time.Now().Add(time.Minute), - }) - - if got := transport.cachedToken("current"); got != "current-token" { - t.Errorf("cachedToken(current) = %q, want %q", got, "current-token") - } - if _, ok := transport.tokens["expired-unused"]; ok { - t.Error("expired unused token was not pruned") - } -} - -func TestTransportDoesNotFollowBearerChallengeOutsideOCIRegistry(t *testing.T) { - tokenRequests := 0 - var server *httptest.Server - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/token" { - tokenRequests++ - _, _ = io.WriteString(w, `{"token":"unexpected"}`) - return - } - w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`) - http.Error(w, "authentication required", http.StatusUnauthorized) - })) - defer server.Close() - - client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)} - resp, err := client.Get(server.URL + "/api/packages") - if err != nil { - t.Fatalf("GET API: %v", err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusUnauthorized { - t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized) - } - if tokenRequests != 0 { - t.Errorf("token requests = %d, want 0", tokenRequests) - } -} - -func TestTransportBoundsTokenCache(t *testing.T) { - transport := NewTransport(http.DefaultTransport, nil) - expiry := time.Now().Add(time.Hour) - for i := range maxCachedTokens + 10 { - transport.cacheToken(fmt.Sprintf("key-%d", i), cachedToken{ - value: "token", - expiresAt: expiry.Add(time.Duration(i) * time.Second), - }) - } - if got := len(transport.tokens); got != maxCachedTokens { - t.Errorf("cached tokens = %d, want %d", got, maxCachedTokens) - } - if got := transport.cachedToken(fmt.Sprintf("key-%d", maxCachedTokens+9)); got != "token" { - t.Error("the newest token was evicted") - } - if got := transport.cachedToken("key-0"); got != "" { - t.Error("the token expiring soonest was kept") - } -} - -func TestTransportBoundsChallengeCache(t *testing.T) { - transport := NewTransport(http.DefaultTransport, nil) - challenge := bearerChallenge{realm: "https://auth.example.test/token", service: "registry"} - for i := range 50 { - transport.rememberChallenge(mustParse(t, fmt.Sprintf("https://registry.example.test/v2/repo-%d/blobs/sha256:abc", i)), challenge) - } - if got := len(transport.challenges); got != 1 { - t.Errorf("challenges for one registry host = %d, want 1", got) - } - for i := range maxCachedChallenges + 10 { - transport.rememberChallenge(mustParse(t, fmt.Sprintf("https://registry-%d.example.test/v2/repo/blobs/sha256:abc", i)), challenge) - } - if got := len(transport.challenges); got != maxCachedChallenges { - t.Errorf("challenges = %d, want %d", got, maxCachedChallenges) - } -} - -func TestTransportCapsTokenLifetime(t *testing.T) { - resp := &http.Response{Body: io.NopCloser(strings.NewReader(`{"token":"t","expires_in":31536000}`))} - _, expiresAt, err := decodeTokenResponse(resp) - if err != nil { - t.Fatal(err) - } - if limit := time.Now().Add(maxTokenLifetime); expiresAt.After(limit) { - t.Errorf("token expires at %v, want no later than %v", expiresAt, limit) - } -} diff --git a/internal/inventory/fuzz_test.go b/internal/inventory/fuzz_test.go deleted file mode 100644 index f7fcadd..0000000 --- a/internal/inventory/fuzz_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package inventory - -import ( - "os" - "testing" -) - -// FuzzParse checks that no uploaded body makes the SBOM parsing panic. -func FuzzParse(f *testing.F) { - if data, err := os.ReadFile("testdata/syft-cyclonedx.json"); err == nil { - f.Add(data) - } - f.Add([]byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[{"type":"library","name":"x","version":"1","purl":"pkg:npm/x@1"}]}`)) - f.Add([]byte(`{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT","packages":[{"SPDXID":"SPDXRef-a","name":"a","versionInfo":"1","externalRefs":[{"referenceCategory":"PACKAGE-MANAGER","referenceType":"purl","referenceLocator":"pkg:conda/a@1?channel=c"}]}]}`)) - f.Add([]byte("SPDXVersion: SPDX-2.3\nPackageName: a\nSPDXID: SPDXRef-a\nExternalRef: PACKAGE-MANAGER purl pkg:deb/debian/a@1\n")) - f.Add([]byte(`xpkg:pypi/X_Y@1`)) - f.Fuzz(func(t *testing.T, data []byte) { - components, _, err := Parse(data) - if err != nil { - return - } - for _, c := range components { - if c.PackagePURL == "" || c.Version == "" || c.Ecosystem == "" { - t.Errorf("component with an empty field: %+v", c) - } - } - }) -} diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go deleted file mode 100644 index c48f387..0000000 --- a/internal/inventory/inventory.go +++ /dev/null @@ -1,190 +0,0 @@ -// Package inventory turns SBOMs uploaded for a project into the package -// versions the proxy records for it, named the way the proxy's handlers name -// the packages they cache, so the two can be joined. -package inventory - -import ( - "errors" - "fmt" - "strings" - - "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/purl" - "github.com/git-pkgs/sbom" - "github.com/git-pkgs/vers" -) - -// MaxProjectNameLength bounds a project name, such as a GitLab project path. -const MaxProjectNameLength = 255 - -// Component is one package version a project uses. -type Component struct { - // Ecosystem is the name the proxy's handlers store the package under, - // e.g. "npm", "composer", "golang" or "alpine". - Ecosystem string - // Name is the ecosystem-native package name, e.g. "@babel/core", - // "symfony/console" or "org.slf4j:slf4j-api". - Name string - // Version is the version as the SBOM gives it; it may be empty. - Version string - // PackagePURL is the unversioned package URL the proxy records the - // package under, so it matches packages.purl. - PackagePURL string -} - -// servedTypes are the PURL types the proxy serves packages of and the -// inventory records. -var servedTypes = map[string]bool{ - "alpm": true, "apk": true, "bsd": true, "cargo": true, "chef": true, - "composer": true, "conan": true, "conda": true, "cran": true, "deb": true, - "gem": true, "golang": true, "hex": true, "julia": true, "maven": true, - "npm": true, "nuget": true, "pub": true, "pypi": true, "rpm": true, - "swift": true, -} - -// ErrUnsupportedPURL is returned for a package URL whose type the proxy -// doesn't serve. -var ErrUnsupportedPURL = errors.New("package type is not served by the proxy") - -// ErrNoCondaChannel is returned for a conda package URL without a channel -// qualifier: the proxy records conda packages per channel. -var ErrNoCondaChannel = errors.New("conda package URL has no channel qualifier") - -// FromPURL maps a package URL to a component, with the package named the way -// the handler for its type records it. Qualifiers and subpath are dropped; -// the version is kept as given. -func FromPURL(s string) (Component, error) { - p, err := purl.Parse(s) - if err != nil { - return Component{}, err - } - if !servedTypes[p.Type] { - return Component{}, fmt.Errorf("%w: %s", ErrUnsupportedPURL, p.Type) - } - ecosystem, name := packageurl.HandlerPackage(p) - if ecosystem == "conda" && !strings.Contains(name, "/") { - return Component{}, ErrNoCondaChannel - } - packagePURL, _ := packageurl.MakeCacheStrings(ecosystem, name, "") - if packagePURL == "" { - return Component{}, fmt.Errorf("package URL %q has no valid package name", s) - } - return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil -} - -// syftMainModule is the property syft sets on modules found in a Go binary, -// naming the module the binary was built from. -const syftMainModule = "syft:metadata:mainModule" - -// unknownVersion is what syft gives a package without a resolved version, -// such as the project's own module in go.mod or the root of a lock file. -const unknownVersion = "UNKNOWN" - -// Parse reads a CycloneDX (JSON or XML) or SPDX SBOM and returns the package -// versions it lists, without duplicates. Packages without a package URL, of a -// type the proxy doesn't serve, without a version, or conda packages without -// a channel are counted in skipped. -// So are the project's own packages: Go modules its binaries were built from, -// and the roots sbom finds for the dependency graph. -func Parse(data []byte) (components []Component, skipped int, err error) { - doc, err := sbom.Parse(data) - if err != nil { - return nil, 0, err - } - ownModules := mainModules(doc.Packages) - roots := map[string]bool{} - for _, id := range doc.ProjectRoots() { - roots[id] = true - } - seen := make(map[[2]string]bool, len(doc.Packages)) - for _, pkg := range doc.Packages { - c, ok := componentOf(pkg) - if !ok || roots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { - skipped++ - continue - } - key := [2]string{c.PackagePURL, c.Version} - if seen[key] { - continue - } - seen[key] = true - components = append(components, c) - } - return components, skipped, nil -} - -func componentOf(pkg sbom.Package) (Component, bool) { - locator := pkg.PURL() - if locator == "" { - return Component{}, false - } - c, err := FromPURL(locator) - if err != nil { - return Component{}, false - } - if c.Version == "" { - c.Version = pkg.Version - } - // A package without a version is the project itself (the module in - // go.mod, the root of package-lock.json), not something it depends on. - if c.Version == "" || c.Version == unknownVersion { - return Component{}, false - } - return c, true -} - -// mainModules returns the Go modules the SBOM's binaries were built from, -// which are the project's own. Module paths are case-sensitive and package -// URLs keep their case, so they are compared as they are. -func mainModules(pkgs []sbom.Package) map[string]bool { - own := map[string]bool{} - for _, pkg := range pkgs { - for _, prop := range pkg.Properties { - if prop.Name == syftMainModule && prop.Value != "" { - own[prop.Value] = true - } - } - } - return own -} - -// ValidateProjectName checks a project name such as "group/sub/repo": one or -// more segments of letters, digits, ".", "_" and "-", none of them "." or -// "..". -func ValidateProjectName(name string) error { - if name == "" { - return errors.New("project name is empty") - } - if len(name) > MaxProjectNameLength { - return fmt.Errorf("project name is longer than %d characters", MaxProjectNameLength) - } - for segment := range strings.SplitSeq(name, "/") { - if segment == "" || segment == "." || segment == ".." { - return fmt.Errorf("project name %q has an empty, \".\" or \"..\" segment", name) - } - for _, r := range segment { - if !validNameRune(r) { - return fmt.Errorf("project name %q may only contain letters, digits, '.', '_', '-' and '/'", name) - } - } - } - return nil -} - -func validNameRune(r rune) bool { - return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '.' || r == '_' || r == '-' -} - -// CompareVersions compares two versions of a package in ecosystem with the -// ecosystem's version rules, returning -1, 0 or 1. -func CompareVersions(ecosystem, a, b string) int { - return vers.CompareWithScheme(a, b, purl.EcosystemToPURLType(ecosystem)) -} - -// Outdated reports whether version is lower than latest. -func Outdated(ecosystem, version, latest string) bool { - if version == "" || latest == "" || version == latest { - return false - } - return CompareVersions(ecosystem, version, latest) < 0 -} diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go deleted file mode 100644 index 89b070b..0000000 --- a/internal/inventory/inventory_test.go +++ /dev/null @@ -1,337 +0,0 @@ -package inventory - -import ( - "errors" - "os" - "slices" - "strings" - "testing" - - "github.com/git-pkgs/proxy/internal/packageurl" -) - -func TestParseSyftCycloneDX(t *testing.T) { - data, err := os.ReadFile("testdata/syft-cyclonedx.json") - if err != nil { - t.Fatal(err) - } - components, skipped, err := Parse(data) - if err != nil { - t.Fatalf("Parse: %v", err) - } - want := []Component{ - {Ecosystem: "npm", Name: "@babel/core", Version: "7.23.0", PackagePURL: "pkg:npm/%40babel/core"}, - {Ecosystem: "alpine", Name: "busybox", Version: "1.36.1-r15", PackagePURL: "pkg:apk/alpine/busybox"}, - {Ecosystem: "golang", Name: "github.com/inconshreveable/mousetrap", Version: "v1.1.0", PackagePURL: "pkg:golang/github.com/inconshreveable/mousetrap"}, - {Ecosystem: "golang", Name: "github.com/spf13/cobra", Version: "v1.8.0", PackagePURL: "pkg:golang/github.com/spf13/cobra"}, - {Ecosystem: "golang", Name: "github.com/spf13/pflag", Version: "v1.0.5", PackagePURL: "pkg:golang/github.com/spf13/pflag"}, - {Ecosystem: "npm", Name: "lodash", Version: "4.17.20", PackagePURL: "pkg:npm/lodash"}, - {Ecosystem: "composer", Name: "monolog/monolog", Version: "3.5.0", PackagePURL: "pkg:composer/monolog/monolog"}, - {Ecosystem: "alpine", Name: "musl", Version: "1.2.4-r2", PackagePURL: "pkg:apk/alpine/musl"}, - {Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1", PackagePURL: "pkg:composer/symfony/console"}, - } - if !slices.Equal(components, want) { - t.Errorf("components =\n%v\nwant\n%v", components, want) - } - // The operating system and the four lock files syft lists carry no PURL, - // and webapp is the root of package-lock.json, the project itself. - if skipped != 6 { - t.Errorf("skipped = %d, want 6", skipped) - } -} - -func TestParseDeduplicatesAndSkipsUnknownTypes(t *testing.T) { - doc := `{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ - {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}, - {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}, - {"type":"library","name":"lodash","version":"4.17.20","purl":"pkg:npm/lodash@4.17.20"}, - {"type":"library","name":"checkout","version":"v4","purl":"pkg:github/actions/checkout@v4"}, - {"type":"library","name":"Django","version":"4.2.0","purl":"pkg:pypi/Django"}, - {"type":"library","name":"guava","version":"32.1.3-jre","purl":"pkg:maven/com.google.guava/guava@32.1.3-jre?type=jar"}, - {"type":"library","name":"libc6","version":"2.36-9","purl":"pkg:deb/debian/libc6@2.36-9?arch=amd64&distro=debian-12"}, - {"type":"library","name":"broken","purl":"not a purl"} - ]}` - components, skipped, err := Parse([]byte(doc)) - if err != nil { - t.Fatal(err) - } - want := []Component{ - {Ecosystem: "npm", Name: "lodash", Version: "4.17.21", PackagePURL: "pkg:npm/lodash"}, - {Ecosystem: "npm", Name: "lodash", Version: "4.17.20", PackagePURL: "pkg:npm/lodash"}, - {Ecosystem: "pypi", Name: "django", Version: "4.2.0", PackagePURL: "pkg:pypi/django"}, - {Ecosystem: "maven", Name: "com.google.guava:guava", Version: "32.1.3-jre", PackagePURL: "pkg:maven/com.google.guava/guava"}, - {Ecosystem: "deb", Name: "libc6", Version: "2.36-9", PackagePURL: "pkg:deb/libc6"}, - } - if !slices.Equal(components, want) { - t.Errorf("components =\n%v\nwant\n%v", components, want) - } - if skipped != 2 { - t.Errorf("skipped = %d, want 2", skipped) - } -} - -func TestParseRejectsNonSBOM(t *testing.T) { - if _, _, err := Parse([]byte(`{"hello":"world"}`)); err == nil { - t.Error("Parse accepted a document that is not an SBOM") - } -} - -func TestFromPURLUnsupported(t *testing.T) { - _, err := FromPURL("pkg:github/actions/checkout") - if !errors.Is(err, ErrUnsupportedPURL) { - t.Errorf("err = %v, want ErrUnsupportedPURL", err) - } -} - -// TestFromPURLMatchesHandlerNames checks, for every PURL type the inventory -// maps, that an SBOM's package URL ends up as the package URL the handler -// records when it caches that package. handlerName is the name the handler -// passes to GetOrFetchArtifactFromURL, taken from the handler's code. -func TestFromPURLMatchesHandlerNames(t *testing.T) { - tests := []struct { - purlType string - sbomPURL string - handlerName string - }{ - {"alpm", "pkg:alpm/arch/glibc@2.39-1?arch=x86_64", "glibc"}, - {"apk", "pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64", "busybox"}, - {"bsd", "pkg:bsd/freebsd/curl@8.7.1", "curl"}, - {"cargo", "pkg:cargo/serde@1.0.200", "serde"}, - {"chef", "pkg:chef/nginx@12.0.0", "nginx"}, - {"composer", "pkg:composer/symfony/console@v6.4.1", "symfony/console"}, - {"conan", "pkg:conan/zlib@1.3.1", "zlib"}, - {"conda", "pkg:conda/numpy@1.26.4?channel=conda-forge&subdir=linux-64", "conda-forge/numpy"}, - {"cran", "pkg:cran/ggplot2@3.5.1", "ggplot2"}, - {"deb", "pkg:deb/debian/libc6@2.36-9?arch=amd64&distro=debian-12", "libc6"}, - {"gem", "pkg:gem/rails@7.1.3", "rails"}, - // The go handler stores the module path with its case; PURLs of - // golang modules are lowercased on both sides. - {"golang", "pkg:golang/github.com/Azure/azure-sdk-for-go@v68.0.0", "github.com/Azure/azure-sdk-for-go"}, - {"hex", "pkg:hex/phoenix@1.7.12", "phoenix"}, - {"julia", "pkg:julia/Example@0.5.3?uuid=7876af07-990d-54b4-ab0e-23690620f79a", "Example"}, - {"maven", "pkg:maven/org.slf4j/slf4j-api@2.0.13?type=jar", "org.slf4j:slf4j-api"}, - {"npm", "pkg:npm/%40babel/core@7.23.0", "@babel/core"}, - // The NuGet handler lowercases the package ID. - {"nuget", "pkg:nuget/Newtonsoft.Json@13.0.3", "newtonsoft.json"}, - {"pub", "pkg:pub/http@1.2.1", "http"}, - // The PyPI handler takes the name from the wheel or sdist file name, - // which spells "." and "-" as "_". - {"pypi", "pkg:pypi/zope.interface@6.4", "zope_interface"}, - {"rpm", "pkg:rpm/fedora/curl@8.6.0-7.fc40?arch=x86_64", "curl"}, - {"swift", "pkg:swift/github.com/apple/swift-argument-parser@1.3.1", "github.com/apple/swift-argument-parser"}, - } - covered := map[string]bool{} - for _, tt := range tests { - t.Run(tt.purlType, func(t *testing.T) { - covered[tt.purlType] = true - c, err := FromPURL(tt.sbomPURL) - if err != nil { - t.Fatalf("FromPURL(%q): %v", tt.sbomPURL, err) - } - want, _ := packageurl.MakeCacheStrings(packageurl.HandlerEcosystem(tt.purlType), tt.handlerName, "") - if want == "" { - t.Fatalf("handler name %q has no package URL", tt.handlerName) - } - if c.PackagePURL != want { - t.Errorf("PackagePURL = %q, want %q as the handler records it", c.PackagePURL, want) - } - }) - } - for purlType := range servedTypes { - if !covered[purlType] { - t.Errorf("no case for PURL type %q", purlType) - } - } -} - -func TestFromPURLNormalizesPyPINames(t *testing.T) { - for _, s := range []string{"pkg:pypi/zope.interface", "pkg:pypi/Zope_Interface", "pkg:pypi/zope--interface", "pkg:pypi/zope._-interface"} { - c, err := FromPURL(s) - if err != nil { - t.Fatal(err) - } - if c.Name != "zope-interface" || c.PackagePURL != "pkg:pypi/zope-interface" { - t.Errorf("FromPURL(%q) = %+v, want zope-interface", s, c) - } - } -} - -func TestFromPURLCondaNeedsChannel(t *testing.T) { - if _, err := FromPURL("pkg:conda/numpy@1.26.4"); !errors.Is(err, ErrNoCondaChannel) { - t.Errorf("err = %v, want ErrNoCondaChannel", err) - } - doc := `{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ - {"type":"library","name":"numpy","version":"1.26.4","purl":"pkg:conda/numpy@1.26.4"}, - {"type":"library","name":"numpy","version":"1.26.4","purl":"pkg:conda/numpy@1.26.4?channel=conda-forge"} - ]}` - components, skipped, err := Parse([]byte(doc)) - if err != nil { - t.Fatal(err) - } - if len(components) != 1 || components[0].Name != "conda-forge/numpy" || skipped != 1 { - t.Errorf("components = %+v, skipped = %d; want conda-forge/numpy and 1 skipped", components, skipped) - } -} - -func TestValidateProjectName(t *testing.T) { - valid := []string{"repo", "group/repo", "group/sub/my_repo-2.0", strings.Repeat("a", MaxProjectNameLength)} - for _, name := range valid { - if err := ValidateProjectName(name); err != nil { - t.Errorf("ValidateProjectName(%q) = %v, want nil", name, err) - } - } - invalid := []string{ - "", "/repo", "repo/", "group//repo", ".", "..", "group/../repo", "group/./repo", - "group/re po", "group/repo?x", "group\\repo", "grüppe/repo", "group/repo%2F", - strings.Repeat("a", MaxProjectNameLength+1), - } - for _, name := range invalid { - if err := ValidateProjectName(name); err == nil { - t.Errorf("ValidateProjectName(%q) = nil, want an error", name) - } - } -} - -func TestOutdated(t *testing.T) { - tests := []struct { - name string - ecosystem string - version string - other string - outdated bool - }{ - {"npm older", "npm", "4.17.20", "4.17.21", true}, - {"npm same", "npm", "4.17.21", "4.17.21", false}, - {"npm newer", "npm", "4.18.0", "4.17.21", false}, - {"composer v prefix", "composer", "v6.4.1", "v6.4.2", true}, - {"golang", "golang", "v1.8.0", "v1.10.0", true}, - {"alpine revisions", "alpine", "1.36.1-r15", "1.36.1-r2", false}, - {"empty version", "npm", "", "1.0.0", false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := Outdated(tt.ecosystem, tt.version, tt.other); got != tt.outdated { - t.Errorf("Outdated = %v, want %v", got, tt.outdated) - } - }) - } -} - -func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { - doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ - {"bom-ref":"mod","type":"library","name":"github.com/acme/app","version":"UNKNOWN","purl":"pkg:golang/github.com/acme/app"}, - {"bom-ref":"bin","type":"library","name":"github.com/acme/app","version":"v0.0.0-20261027162038-a3e9dd9fd3f4+dirty","purl":"pkg:golang/github.com/acme/app@v0.0.0-20261027162038-a3e9dd9fd3f4%2Bdirty", - "properties":[{"name":"syft:metadata:mainModule","value":"github.com/acme/app"}]}, - {"bom-ref":"cobra","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0", - "properties":[{"name":"syft:metadata:mainModule","value":"github.com/acme/app"}]}, - {"bom-ref":"root","type":"library","name":"app-frontend","version":"2.1.0","purl":"pkg:npm/app-frontend@2.1.0", - "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]}, - {"bom-ref":"nover","type":"library","name":"app-tools","purl":"pkg:npm/app-tools"}, - {"bom-ref":"jsdom","type":"library","name":"jsdom","version":"30.1.3","purl":"pkg:npm/jsdom@30.1.3", - "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]} - ],"dependencies":[{"ref":"root","dependsOn":["jsdom"]}]}` - components, skipped, err := Parse([]byte(doc)) - if err != nil { - t.Fatal(err) - } - want := []Component{ - {Ecosystem: "golang", Name: "github.com/spf13/cobra", Version: "v1.8.0", PackagePURL: "pkg:golang/github.com/spf13/cobra"}, - {Ecosystem: "npm", Name: "jsdom", Version: "30.1.3", PackagePURL: "pkg:npm/jsdom"}, - } - if !slices.Equal(components, want) { - t.Errorf("components =\n%v\nwant\n%v", components, want) - } - if skipped != 4 { - t.Errorf("skipped = %d, want 4", skipped) - } -} - -func TestParseKeepsTopLevelDependenciesOfLockFiles(t *testing.T) { - // A poetry.lock doesn't list the project, so requests is a direct - // dependency, not the project's own entry. - doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ - {"bom-ref":"requests","type":"library","name":"requests","version":"2.32.3","purl":"pkg:pypi/requests@2.32.3", - "properties":[{"name":"syft:package:metadataType","value":"python-poetry-lock-entry"},{"name":"syft:location:0:path","value":"/poetry.lock"}]}, - {"bom-ref":"urllib3","type":"library","name":"urllib3","version":"2.2.3","purl":"pkg:pypi/urllib3@2.2.3", - "properties":[{"name":"syft:package:metadataType","value":"python-poetry-lock-entry"},{"name":"syft:location:0:path","value":"/poetry.lock"}]} - ],"dependencies":[{"ref":"requests","dependsOn":["urllib3"]}]}` - components, skipped, err := Parse([]byte(doc)) - if err != nil { - t.Fatal(err) - } - if len(components) != 2 || skipped != 0 { - t.Errorf("components = %+v, skipped = %d; want requests and urllib3, 0 skipped", components, skipped) - } -} - -func TestParseKeepsOSPackagesNothingDependsOn(t *testing.T) { - // syft's CycloneDX of an image: metadata.component is the image, with - // no edges, and busybox is something nothing else depends on. - doc := `{"bomFormat":"CycloneDX","specVersion":"1.6", - "metadata":{"component":{"bom-ref":"image","type":"container","name":"alpine:3.20"}}, - "components":[ - {"bom-ref":"busybox","type":"library","name":"busybox","version":"1.36.1-r29","purl":"pkg:apk/alpine/busybox@1.36.1-r29?distro=alpine-3.20.3", - "properties":[{"name":"syft:package:metadataType","value":"apk-db-entry"}]}, - {"bom-ref":"musl","type":"library","name":"musl","version":"1.2.5-r0","purl":"pkg:apk/alpine/musl@1.2.5-r0?distro=alpine-3.20.3", - "properties":[{"name":"syft:package:metadataType","value":"apk-db-entry"}]} - ],"dependencies":[{"ref":"busybox","dependsOn":["musl"]}]}` - components, skipped, err := Parse([]byte(doc)) - if err != nil { - t.Fatal(err) - } - if len(components) != 2 || skipped != 0 { - t.Errorf("components = %+v, skipped = %d; want busybox and musl, 0 skipped", components, skipped) - } -} - -func TestParseSkipsTheDeclaredRoot(t *testing.T) { - doc := `{"bomFormat":"CycloneDX","specVersion":"1.6", - "metadata":{"component":{"bom-ref":"app","type":"application","name":"app","version":"1.0.0","purl":"pkg:npm/app@1.0.0"}}, - "components":[ - {"bom-ref":"lodash","type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"} - ],"dependencies":[{"ref":"app","dependsOn":["lodash"]}]}` - components, _, err := Parse([]byte(doc)) - if err != nil { - t.Fatal(err) - } - if len(components) != 1 || components[0].Name != "lodash" { - t.Errorf("components = %+v, want only lodash", components) - } -} - -func TestParseSkipsMixedCaseMainModule(t *testing.T) { - // syft keeps the module path's case in mainModule, and so does the - // package URL. - doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ - {"bom-ref":"bin","type":"library","name":"github.com/Azure/foo","version":"v1.2.0","purl":"pkg:golang/github.com/Azure/foo@v1.2.0", - "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]}, - {"bom-ref":"cobra","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0", - "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]} - ]}` - components, skipped, err := Parse([]byte(doc)) - if err != nil { - t.Fatal(err) - } - if len(components) != 1 || components[0].Name != "github.com/spf13/cobra" || skipped != 1 { - t.Errorf("components = %+v, skipped = %d; want only cobra, 1 skipped", components, skipped) - } -} - -func TestParseMainModuleKeepsCase(t *testing.T) { - // Go module paths are case-sensitive, and package URLs keep their case, - // so github.com/azure/foo is another module than the main module - // github.com/Azure/foo, and a dependency. - doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ - {"bom-ref":"bin","type":"library","name":"github.com/Azure/foo","version":"v1.2.0","purl":"pkg:golang/github.com/Azure/foo@v1.2.0", - "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]}, - {"bom-ref":"other","type":"library","name":"github.com/azure/foo","version":"v0.1.0","purl":"pkg:golang/github.com/azure/foo@v0.1.0", - "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]} - ]}` - components, skipped, err := Parse([]byte(doc)) - if err != nil { - t.Fatal(err) - } - if len(components) != 1 || components[0].Name != "github.com/azure/foo" || skipped != 1 { - t.Errorf("components = %+v, skipped = %d; want only github.com/azure/foo, 1 skipped", components, skipped) - } -} diff --git a/internal/inventory/testdata/syft-cyclonedx.json b/internal/inventory/testdata/syft-cyclonedx.json deleted file mode 100644 index 7647d68..0000000 --- a/internal/inventory/testdata/syft-cyclonedx.json +++ /dev/null @@ -1 +0,0 @@ -{"$schema":"http://cyclonedx.org/schema/bom-1.7.schema.json","bomFormat":"CycloneDX","specVersion":"1.7","serialNumber":"urn:uuid:d69cf664-a43e-4d65-93e3-e43e24d33c0e","version":1,"metadata":{"timestamp":"2026-10-27T19:17:53+01:00","tools":{"components":[{"type":"application","author":"anchore","name":"syft","version":"1.55.0"}]},"component":{"bom-ref":"af63bd4c8601b7f1","type":"file","name":"."}},"components":[{"bom-ref":"pkg:npm/%40babel/core@7.23.0?package-id=7b01fd7a9ce63128","type":"library","name":"@babel/core","version":"7.23.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:\\@babel\\/core:\\@babel\\/core:7.23.0:*:*:*:*:*:*:*","purl":"pkg:npm/%40babel/core@7.23.0","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64&distro=alpine-3.19.1&package-id=464f552f6c89c192","type":"library","publisher":"Sören Tempel ","name":"busybox","version":"1.36.1-r15","description":"Size optimized toolbox of many common UNIX utilities","licenses":[{"license":{"id":"GPL-2.0-only"}}],"cpe":"cpe:2.3:a:busybox:busybox:1.36.1-r15:*:*:*:*:*:*:*","purl":"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64&distro=alpine-3.19.1","externalReferences":[{"url":"https://busybox.net/","type":"distribution"}],"properties":[{"name":"syft:package:foundBy","value":"apk-db-cataloger"},{"name":"syft:package:type","value":"apk"},{"name":"syft:package:metadataType","value":"apk-db-entry"},{"name":"syft:location:0:path","value":"/lib/apk/db/installed"},{"name":"syft:metadata:gitCommitOfApkPort","value":"1b0b2f6e7d0e6a5b4c3d2e1f0a9b8c7d6e5f4a3b"},{"name":"syft:metadata:installedSize","value":"950272"},{"name":"syft:metadata:originPackage","value":"busybox"},{"name":"syft:metadata:pullChecksum","value":"Q1Xy0qB0j7Y7uQ2kGmWbN8vT6Ff0A4E="},{"name":"syft:metadata:pullDependencies:0","value":"so:libc.musl-x86_64.so.1"},{"name":"syft:metadata:size","value":"509268"}]},{"bom-ref":"pkg:golang/github.com/inconshreveable/mousetrap@v1.1.0?package-id=f6300927a9f6b106","type":"library","name":"github.com/inconshreveable/mousetrap","version":"v1.1.0","cpe":"cpe:2.3:a:inconshreveable:mousetrap:v1.1.0:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/inconshreveable/mousetrap@v1.1.0","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8="}]},{"bom-ref":"pkg:golang/github.com/spf13/cobra@v1.8.0?package-id=81b06fbcbe87f414","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","cpe":"cpe:2.3:a:spf13:cobra:v1.8.0:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0="}]},{"bom-ref":"pkg:golang/github.com/spf13/pflag@v1.0.5?package-id=163f67459b76e534","type":"library","name":"github.com/spf13/pflag","version":"v1.0.5","cpe":"cpe:2.3:a:spf13:pflag:v1.0.5:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/spf13/pflag@v1.0.5","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRPuqXzS0rkr6qZw="}]},{"bom-ref":"pkg:npm/lodash@4.17.20?package-id=ece3658907628a6b","type":"library","name":"lodash","version":"4.17.20","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:lodash:lodash:4.17.20:*:*:*:*:node.js:*:*","purl":"pkg:npm/lodash@4.17.20","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"pkg:composer/monolog/monolog@3.5.0?package-id=3abfdcdd00fd8d2f","type":"library","name":"monolog/monolog","version":"3.5.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:monolog\\/monolog:monolog\\/monolog:3.5.0:*:*:*:*:*:*:*","purl":"pkg:composer/monolog/monolog@3.5.0","properties":[{"name":"syft:package:foundBy","value":"php-composer-lock-cataloger"},{"name":"syft:package:language","value":"php"},{"name":"syft:package:type","value":"php-composer"},{"name":"syft:package:metadataType","value":"php-composer-lock-entry"},{"name":"syft:location:0:path","value":"/composer.lock"}]},{"bom-ref":"pkg:apk/alpine/musl@1.2.4-r2?arch=x86_64&distro=alpine-3.19.1&package-id=9f0e3d80c35c10bc","type":"library","publisher":"Timo Teräs ","name":"musl","version":"1.2.4-r2","description":"the musl c library (libc) implementation","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:musl-libc:musl:1.2.4-r2:*:*:*:*:*:*:*","purl":"pkg:apk/alpine/musl@1.2.4-r2?arch=x86_64&distro=alpine-3.19.1","externalReferences":[{"url":"https://musl.libc.org/","type":"distribution"}],"properties":[{"name":"syft:package:foundBy","value":"apk-db-cataloger"},{"name":"syft:package:type","value":"apk"},{"name":"syft:package:metadataType","value":"apk-db-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:musl_libc:musl:1.2.4-r2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:musl:musl:1.2.4-r2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/lib/apk/db/installed"},{"name":"syft:metadata:gitCommitOfApkPort","value":"4b2f0d7b4e0e8a4b0e1b1b4d2b8a7a3d1a2f8e0c"},{"name":"syft:metadata:installedSize","value":"647168"},{"name":"syft:metadata:originPackage","value":"musl"},{"name":"syft:metadata:pullChecksum","value":"Q1N9CgYbNRm8SgkqbXqJhCAc6iqXo="},{"name":"syft:metadata:size","value":"407383"}]},{"bom-ref":"pkg:composer/symfony/console@v6.4.1?package-id=767524bf09da7da7","type":"library","name":"symfony/console","version":"v6.4.1","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:symfony\\/console:symfony\\/console:v6.4.1:*:*:*:*:*:*:*","purl":"pkg:composer/symfony/console@v6.4.1","properties":[{"name":"syft:package:foundBy","value":"php-composer-lock-cataloger"},{"name":"syft:package:language","value":"php"},{"name":"syft:package:type","value":"php-composer"},{"name":"syft:package:metadataType","value":"php-composer-lock-entry"},{"name":"syft:location:0:path","value":"/composer.lock"}]},{"bom-ref":"pkg:npm/webapp@1.0.0?package-id=982a05090388c92d","type":"library","name":"webapp","version":"1.0.0","cpe":"cpe:2.3:a:webapp:webapp:1.0.0:*:*:*:*:*:*:*","purl":"pkg:npm/webapp@1.0.0","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"os:alpine@3.19.1","type":"operating-system","name":"alpine","version":"3.19.1","description":"Alpine Linux v3.19","swid":{"tagId":"alpine","name":"alpine","version":"3.19.1"},"properties":[{"name":"syft:distro:extendedSupport","value":"false"},{"name":"syft:distro:id","value":"alpine"},{"name":"syft:distro:prettyName","value":"Alpine Linux v3.19"},{"name":"syft:distro:versionID","value":"3.19.1"}]},{"bom-ref":"c8bb62e6bded633e","type":"file","name":"/builds/acme/webapp/composer.lock","hashes":[{"alg":"SHA-1","content":"53c59ca459a8ff42918b5ed4fdcea1d8ba98432f"},{"alg":"SHA-256","content":"63738301cc7186a6645f00ae0a63ae6347e04552489f3cc70e92200463f5b4ac"}]},{"bom-ref":"3fc5a8d3d86e9790","type":"file","name":"/builds/acme/webapp/go.mod","hashes":[{"alg":"SHA-1","content":"c67c41265c415c41049b31f850b4497898eff1fd"},{"alg":"SHA-256","content":"f2303d4139e491238d8cb9a6c448e4c366b82bae3a646164465387df76e44f32"}]},{"bom-ref":"9f5aca292136191a","type":"file","name":"/builds/acme/webapp/lib/apk/db/installed","hashes":[{"alg":"SHA-1","content":"b2ebf78ea604d3a76359a262afd401ab966a36a1"},{"alg":"SHA-256","content":"d363cb96812dc6eb1d6f4c00879c2564cdbff238b6c1d0a37701d2adedd326e0"}]},{"bom-ref":"fd71c2238fc07657","type":"file","name":"/builds/acme/webapp/package-lock.json","hashes":[{"alg":"SHA-1","content":"fc26eadd11405354163eb0f53ffd2bb8cb56c5e7"},{"alg":"SHA-256","content":"69094786dd997053d1c7a81d4349bb5b608c08b576e67360f07cdc2726a041ea"}]}],"dependencies":[{"ref":"pkg:npm/webapp@1.0.0?package-id=982a05090388c92d","dependsOn":["pkg:npm/%40babel/core@7.23.0?package-id=7b01fd7a9ce63128","pkg:npm/lodash@4.17.20?package-id=ece3658907628a6b"]}]} diff --git a/internal/jsonscan/jsonscan.go b/internal/jsonscan/jsonscan.go deleted file mode 100644 index 7fcade7..0000000 --- a/internal/jsonscan/jsonscan.go +++ /dev/null @@ -1,309 +0,0 @@ -// Package jsonscan walks a JSON document in place, reporting where -// object members sit as offsets into the original bytes. It lets a caller -// read or replace one value in a large metadata document without decoding the -// rest of it into Go values, which for an npm packument costs many times the -// document's size. -// -// It checks structure only as far as it needs to find its way through -// the document. Callers that copy unvisited bytes into a response validate the -// whole document first with json.Valid. -package jsonscan - -import ( - "bytes" - "encoding/json" - "errors" -) - -// ErrMalformed is returned when the document is not well-formed JSON. -var ErrMalformed = errors.New("malformed JSON") - -// ErrNotObject is returned when a value expected to be an object is not. -var ErrNotObject = errors.New("JSON value is not an object") - -// ErrStop is returned by a callback to end a walk early once it has -// found what it was looking for. -var ErrStop = errors.New("stop scanning") - -// Member is one member of a JSON object as offsets into the bytes that -// were scanned. The key span includes its quotes. -type Member struct { - KeyStart, KeyEnd int - ValStart, ValEnd int -} - -func (m Member) Key(data []byte) []byte { return data[m.KeyStart:m.KeyEnd] } -func (m Member) Value(data []byte) []byte { return data[m.ValStart:m.ValEnd] } - -func skipJSONSpace(data []byte, i int) int { - for i < len(data) { - switch data[i] { - case ' ', '\t', '\n', '\r': - i++ - default: - return i - } - } - return i -} - -// skipJSONString returns the index just past the string starting at data[i]. -func skipJSONString(data []byte, i int) (int, error) { - for j := i + 1; j < len(data); j++ { - switch data[j] { - case '\\': - j++ - case '"': - return j + 1, nil - } - } - return 0, ErrMalformed -} - -// skipJSONValue returns the index just past the value starting at data[i]. -func skipJSONValue(data []byte, i int) (int, error) { - if i >= len(data) { - return 0, ErrMalformed - } - switch data[i] { - case '"': - return skipJSONString(data, i) - case '{', '[': - depth := 0 - for j := i; j < len(data); j++ { - switch data[j] { - case '"': - end, err := skipJSONString(data, j) - if err != nil { - return 0, err - } - j = end - 1 - case '{', '[': - depth++ - case '}', ']': - depth-- - if depth == 0 { - return j + 1, nil - } - } - } - return 0, ErrMalformed - case '}', ']', ',', ':': - return 0, ErrMalformed - default: - // A number, true, false or null runs to the next delimiter. - j := i - for j < len(data) { - switch data[j] { - case ',', '}', ']', ' ', '\t', '\n', '\r': - return j, nil - } - j++ - } - return j, nil - } -} - -// EachMember calls fn for each member of the object obj holds, in -// document order. It returns ErrNotObject if obj is not an object, and -// stops early with fn's error if fn returns one. -func EachMember(obj []byte, fn func(Member) error) error { - i := skipJSONSpace(obj, 0) - if i >= len(obj) || obj[i] != '{' { - return ErrNotObject - } - i = skipJSONSpace(obj, i+1) - if i < len(obj) && obj[i] == '}' { - return nil - } - for { - if i >= len(obj) || obj[i] != '"' { - return ErrMalformed - } - keyEnd, err := skipJSONString(obj, i) - if err != nil { - return err - } - colon := skipJSONSpace(obj, keyEnd) - if colon >= len(obj) || obj[colon] != ':' { - return ErrMalformed - } - valStart := skipJSONSpace(obj, colon+1) - valEnd, err := skipJSONValue(obj, valStart) - if err != nil { - return err - } - if err := fn(Member{KeyStart: i, KeyEnd: keyEnd, ValStart: valStart, ValEnd: valEnd}); err != nil { - return err - } - next := skipJSONSpace(obj, valEnd) - if next >= len(obj) { - return ErrMalformed - } - switch obj[next] { - case ',': - i = skipJSONSpace(obj, next+1) - case '}': - return nil - default: - return ErrMalformed - } - } -} - -// EachElement calls fn with each element of the array arr holds, in -// order, and stops early with fn's error if fn returns one. -func EachElement(arr []byte, fn func(element []byte) error) error { - i := skipJSONSpace(arr, 0) - if i >= len(arr) || arr[i] != '[' { - return ErrMalformed - } - i = skipJSONSpace(arr, i+1) - if i < len(arr) && arr[i] == ']' { - return nil - } - for { - end, err := skipJSONValue(arr, i) - if err != nil { - return err - } - if err := fn(arr[i:end]); err != nil { - return err - } - next := skipJSONSpace(arr, end) - if next >= len(arr) { - return ErrMalformed - } - switch arr[next] { - case ',': - i = skipJSONSpace(arr, next+1) - case ']': - return nil - default: - return ErrMalformed - } - } -} - -// RewriteMembers copies the object obj holds to out byte for byte, -// except that rewrite may write a member's value itself. It returns true -// when it did; otherwise the original value is copied. -func RewriteMembers(out *bytes.Buffer, obj []byte, rewrite func(out *bytes.Buffer, m Member) (bool, error)) error { - copied := 0 - err := EachMember(obj, func(m Member) error { - out.Write(obj[copied:m.ValStart]) - copied = m.ValStart - rewritten, err := rewrite(out, m) - if rewritten { - copied = m.ValEnd - } - return err - }) - out.Write(obj[copied:]) - return err -} - -// Key decodes a quoted key as EachMember reports it. -func Key(raw []byte) (string, error) { - if bytes.IndexByte(raw, '\\') < 0 { - return string(raw[1 : len(raw)-1]), nil - } - var key string - err := json.Unmarshal(raw, &key) - return key, err -} - -// KeyIs reports whether a quoted key decodes to name. -func KeyIs(raw []byte, name string) bool { - if bytes.IndexByte(raw, '\\') < 0 { - return string(raw[1:len(raw)-1]) == name - } - key, err := Key(raw) - return err == nil && key == name -} - -// StringIs reports whether raw is a JSON string that decodes to s. -func StringIs(raw []byte, s string) bool { - return len(raw) >= 2 && raw[0] == '"' && KeyIs(raw, s) -} - -// FindMember returns the member of obj named name. When the key repeats, -// the last one wins, as it does for JSON.parse and encoding/json. -func FindMember(obj []byte, name string) (Member, bool, error) { - var found Member - ok := false - err := EachMember(obj, func(m Member) error { - if KeyIs(m.Key(obj), name) { - found, ok = m, true - } - return nil - }) - return found, ok, err -} - -// Lookup follows path through nested objects in doc and returns the -// value at its end. It returns nil and no error when a key along the path is -// missing or names something other than an object. -func Lookup(doc []byte, path ...string) ([]byte, error) { - value := doc - for _, name := range path { - m, ok, err := FindMember(value, name) - if errors.Is(err, ErrNotObject) { - return nil, nil - } - if err != nil || !ok { - return nil, err - } - value = m.Value(value) - } - return value, nil -} - -// LookupString is Lookup for a string value. ok is false when the -// value is missing or is not a string. -func LookupString(doc []byte, path ...string) (string, bool, error) { - raw, err := Lookup(doc, path...) - if err != nil || len(raw) == 0 || raw[0] != '"' { - return "", false, err - } - var s string - if err := json.Unmarshal(raw, &s); err != nil { - return "", false, err - } - return s, true, nil -} - -// WriteFilteredObject writes the object obj holds, keeping the members -// whose decoded key keep accepts, followed by extra string members. -func WriteFilteredObject(out *bytes.Buffer, obj []byte, keep func(string) bool, extra [][2]string) error { - out.WriteByte('{') - first := true - err := EachMember(obj, func(m Member) error { - key, err := Key(m.Key(obj)) - if err != nil { - return err - } - if !keep(key) { - return nil - } - if !first { - out.WriteByte(',') - } - first = false - out.Write(obj[m.KeyStart:m.ValEnd]) - return nil - }) - for _, member := range extra { - if !first { - out.WriteByte(',') - } - first = false - key, _ := json.Marshal(member[0]) - value, _ := json.Marshal(member[1]) - out.Write(key) - out.WriteByte(':') - out.Write(value) - } - out.WriteByte('}') - return err -} diff --git a/internal/jsonscan/jsonscan_test.go b/internal/jsonscan/jsonscan_test.go deleted file mode 100644 index a14ac85..0000000 --- a/internal/jsonscan/jsonscan_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package jsonscan - -import ( - "bytes" - "errors" - "testing" -) - -func TestLookupJSONString(t *testing.T) { - doc := []byte(`{ - "name": "demo", - "versions": { - "1.0.0": {"dist": {"tarball": "https://example.com/a.tgz", "shasum": "x"}}, - "2.0.0": {"readme": "a } tricky \" string ] with {brackets}", "dist": {"tarball": "https://example.com/b.tgz"}}, - "3.0.0": "not an object", - "4.0.0": {"dist": {"tarball": 42}} - }, - "time": {"1.0.0": "2020-01-01T00:00:00Z", "1.0.0": "2021-01-01T00:00:00Z"}, - "n": -1.5e3, "t": true, "z": null, "list": [1, {"a": []}, "]"] - }`) - - cases := []struct { - path []string - want string - wantOK bool - }{ - {[]string{"versions", "1.0.0", "dist", "tarball"}, "https://example.com/a.tgz", true}, - {[]string{"versions", "2.0.0", "dist", "tarball"}, "https://example.com/b.tgz", true}, - {[]string{"versions", "3.0.0", "dist", "tarball"}, "", false}, - {[]string{"versions", "4.0.0", "dist", "tarball"}, "", false}, - {[]string{"versions", "9.9.9", "dist", "tarball"}, "", false}, - {[]string{"time", "1.0.0"}, "2021-01-01T00:00:00Z", true}, // last duplicate wins - {[]string{"name"}, "demo", true}, - {[]string{"name", "x"}, "", false}, - } - for _, c := range cases { - got, ok, err := LookupString(doc, c.path...) - if err != nil || ok != c.wantOK || got != c.want { - t.Errorf("lookup %v = %q, %v, %v; want %q, %v", c.path, got, ok, err, c.want, c.wantOK) - } - } -} - -func TestLookupJSONEscapedKey(t *testing.T) { - doc := []byte(`{"versions": {"1.0.0": {"dist": {"tarball": "u"}}}}`) - got, ok, err := LookupString(doc, "versions", "1.0.0", "dist", "tarball") - if err != nil || !ok || got != "u" { - t.Errorf("lookup = %q, %v, %v", got, ok, err) - } -} - -func TestLookupJSONMalformed(t *testing.T) { - for _, doc := range []string{ - `{"versions": {"1.0.0": {"dist": `, - `{"versions" {}}`, - `{"versions": {"a": 1 "b": 2}}`, - `{"a": "unterminated}`, - `{"a": [1, 2}`, - } { - _, _, err := LookupString([]byte(doc), "versions", "1.0.0", "dist", "tarball") - if !errors.Is(err, ErrMalformed) { - t.Errorf("%s: err = %v, want ErrMalformed", doc, err) - } - } -} - -func TestForEachJSONMemberOffsets(t *testing.T) { - doc := []byte(` { "a" : 1 , "b":{"c":[true,null]} ,"d":"x\"y" } `) - var got [][2]string - err := EachMember(doc, func(m Member) error { - got = append(got, [2]string{string(m.Key(doc)), string(m.Value(doc))}) - return nil - }) - want := [][2]string{{`"a"`, `1`}, {`"b"`, `{"c":[true,null]}`}, {`"d"`, `"x\"y"`}} - if err != nil || len(got) != len(want) { - t.Fatalf("members = %v, %v", got, err) - } - for i := range want { - if got[i] != want[i] { - t.Errorf("member %d = %v, want %v", i, got[i], want[i]) - } - } - - if err := EachMember([]byte(`[1]`), func(Member) error { return nil }); !errors.Is(err, ErrNotObject) { - t.Errorf("array: err = %v, want ErrNotObject", err) - } - if err := EachMember([]byte(`{}`), func(Member) error { t.Error("called for empty object"); return nil }); err != nil { - t.Errorf("empty object: err = %v", err) - } -} - -func TestWriteFilteredJSONObject(t *testing.T) { - var out bytes.Buffer - err := WriteFilteredObject(&out, []byte(`{"a": 1, "b": {"x": 2}, "c": 3}`), func(k string) bool { return k != "b" }, nil) - if err != nil || out.String() != `{"a": 1,"c": 3}` { - t.Errorf("filtered = %s, %v", out.String(), err) - } -} - -func TestForEachJSONElement(t *testing.T) { - var got []string - err := EachElement([]byte(` [ 1, "a,]", {"b": [2]} ,[] ] `), func(e []byte) error { - got = append(got, string(e)) - return nil - }) - want := []string{`1`, `"a,]"`, `{"b": [2]}`, `[]`} - if err != nil || len(got) != len(want) { - t.Fatalf("elements = %q, %v; want %q", got, err, want) - } - for i := range want { - if got[i] != want[i] { - t.Errorf("element %d = %s, want %s", i, got[i], want[i]) - } - } - - if err := EachElement([]byte(`[]`), func([]byte) error { t.Error("called for empty array"); return nil }); err != nil { - t.Errorf("empty array: err = %v", err) - } - for _, bad := range []string{`{}`, `[1,`, `[1 2]`, `[1`} { - if err := EachElement([]byte(bad), func([]byte) error { return nil }); err == nil { - t.Errorf("%s: expected an error", bad) - } - } -} - -func TestRewriteJSONMembers(t *testing.T) { - obj := []byte(`{ "a": 1, "b": [2], "c" : 3 }`) - var out bytes.Buffer - err := RewriteMembers(&out, obj, func(out *bytes.Buffer, m Member) (bool, error) { - if !KeyIs(m.Key(obj), "b") { - return false, nil - } - out.WriteString(`"B"`) - return true, nil - }) - if want := `{ "a": 1, "b": "B", "c" : 3 }`; err != nil || out.String() != want { - t.Errorf("rewritten = %s, %v; want %s", out.String(), err, want) - } -} - -func TestJSONStringIs(t *testing.T) { - for raw, want := range map[string]bool{ - `"~dev"`: true, - `"~\u0064ev"`: true, - `"~dev2"`: false, - `{"~dev":1}`: false, - `"`: false, - ``: false, - `["~dev"]`: false, - } { - if got := StringIs([]byte(raw), "~dev"); got != want { - t.Errorf("StringIs(%s) = %v, want %v", raw, got, want) - } - } -} diff --git a/internal/jsonstream/jsonstream.go b/internal/jsonstream/jsonstream.go deleted file mode 100644 index cf213ce..0000000 --- a/internal/jsonstream/jsonstream.go +++ /dev/null @@ -1,172 +0,0 @@ -// Package jsonstream copies JSON objects from a reader to a writer one member -// at a time, so a document can be filtered without holding it in memory. -// Values that are kept are written with their original bytes. -package jsonstream - -import ( - "bytes" - "encoding/json/jsontext" - "errors" - "fmt" - "io" -) - -// ErrNotObject is returned when a value expected to be an object is not. -var ErrNotObject = errors.New("JSON value is not an object") - -var ( - errConsumed = errors.New("jsonstream: member value already consumed") - errUnconsumed = errors.New("jsonstream: Write before the member value was read") -) - -// Member is one member of an object being copied. Its value must be consumed -// exactly once, with Raw, Skip, Copy or Object. -type Member struct { - Key string - rawKey []byte - dec *jsontext.Decoder - out *objectWriter - consumed bool -} - -// Raw reads the member's value without writing it. Follow it with Write to -// keep the member. The bytes are only valid until the next member is read. -func (m *Member) Raw() ([]byte, error) { - if m.consumed { - return nil, errConsumed - } - m.consumed = true - return m.dec.ReadValue() -} - -// Skip reads past the member's value, leaving the member out. -func (m *Member) Skip() error { - if m.consumed { - return errConsumed - } - m.consumed = true - return m.dec.SkipValue() -} - -// Write writes the member with value as its raw JSON value. -func (m *Member) Write(value []byte) error { - if !m.consumed { - return errUnconsumed - } - if err := m.out.key(m.rawKey); err != nil { - return err - } - _, err := m.out.w.Write(value) - return err -} - -// Copy writes the member with its value unchanged. -func (m *Member) Copy() error { - raw, err := m.Raw() - if err != nil { - return err - } - return m.Write(raw) -} - -// Object writes the member and copies its value, which must be an object, -// calling fn for each of its members. A null value is written as null. -func (m *Member) Object(fn func(*Member) error) error { - if m.consumed { - return errConsumed - } - m.consumed = true - if err := m.out.key(m.rawKey); err != nil { - return err - } - return copyObject(m.dec, m.out.w, fn, true) -} - -// CopyObject copies the object read from r to w, calling fn for each member. -// fn decides what is written by how it consumes the member's value; a member -// whose value fn leaves unconsumed is copied unchanged. As with -// encoding/json, of repeated names each is passed to fn, and invalid UTF-8 -// is accepted. -func CopyObject(r io.Reader, w io.Writer, fn func(*Member) error) error { - dec := jsontext.NewDecoder(r, jsontext.AllowDuplicateNames(true), jsontext.AllowInvalidUTF8(true)) - if err := copyObject(dec, w, fn, false); err != nil { - return err - } - _, err := dec.ReadToken() - switch err { - case io.EOF: - return nil - case nil: - return errors.New("unexpected data after the top-level object") - } - return err -} - -func copyObject(dec *jsontext.Decoder, w io.Writer, fn func(*Member) error, allowNull bool) error { - tok, err := dec.ReadToken() - if err != nil { - return err - } - if tok.Kind() == 'n' && allowNull { - _, err := io.WriteString(w, "null") - return err - } - if tok.Kind() != '{' { - return ErrNotObject - } - if _, err := io.WriteString(w, "{"); err != nil { - return err - } - out := &objectWriter{w: w} - m := &Member{dec: dec, out: out} - // keyBuf holds the current key; ReadValue's bytes don't outlast fn. - var keyBuf []byte - for dec.PeekKind() != '}' { - // The key is read raw, so it is written with its own bytes. - rawKey, err := dec.ReadValue() - if err != nil { - return err - } - if rawKey.Kind() != '"' { - return fmt.Errorf("object key is %v, not a string", rawKey.Kind()) - } - keyBuf = append(keyBuf[:0], rawKey...) - // The decoder has checked the key, so the only error left is invalid - // UTF-8, which AppendUnquote reports after replacing it with U+FFFD, - // as encoding/json does. - key, _ := jsontext.AppendUnquote(nil, keyBuf) - *m = Member{Key: string(key), rawKey: keyBuf, dec: dec, out: out} - if err := fn(m); err != nil { - return err - } - if !m.consumed { - if err := m.Copy(); err != nil { - return err - } - } - } - if _, err := dec.ReadToken(); err != nil { - return err - } - _, err = io.WriteString(w, "}") - return err -} - -type objectWriter struct { - w io.Writer - wrote bool - buf bytes.Buffer -} - -// key writes the separator and the key's bytes for the next member. -func (o *objectWriter) key(rawKey []byte) error { - o.buf.Reset() - if o.wrote { - o.buf.WriteByte(',') - } - o.buf.Write(rawKey) - o.buf.WriteByte(':') - o.wrote = true - _, err := o.w.Write(o.buf.Bytes()) - return err -} diff --git a/internal/jsonstream/jsonstream_test.go b/internal/jsonstream/jsonstream_test.go deleted file mode 100644 index cd201ae..0000000 --- a/internal/jsonstream/jsonstream_test.go +++ /dev/null @@ -1,202 +0,0 @@ -package jsonstream - -import ( - "bytes" - "encoding/json" - "errors" - "slices" - "strings" - "testing" -) - -func copyString(t *testing.T, in string, fn func(*Member) error) (string, error) { - t.Helper() - var out bytes.Buffer - err := CopyObject(strings.NewReader(in), &out, fn) - return out.String(), err -} - -func TestCopyObjectKeepsValueBytes(t *testing.T) { - in := `{"a": 1.50, "b" : {"x": "<&>", "y":[1, 2]}, "c":null, "d":"é"}` - got, err := copyString(t, in, func(m *Member) error { return nil }) - if err != nil { - t.Fatal(err) - } - if want := `{"a":1.50,"b":{"x": "<&>", "y":[1, 2]},"c":null,"d":"é"}`; got != want { - t.Errorf("got %s\nwant %s", got, want) - } -} - -func TestCopyObjectFiltersNested(t *testing.T) { - in := `{"info":{"subdir":"noarch"},"packages":{"keep":{"n":1},"drop":{"n":2},"also":{"n":3}},"tail":[]}` - got, err := copyString(t, in, func(m *Member) error { - if m.Key != "packages" { - return m.Copy() - } - return m.Object(func(entry *Member) error { - raw, err := entry.Raw() - if err != nil { - return err - } - if entry.Key == "drop" { - return nil - } - return entry.Write(raw) - }) - }) - if err != nil { - t.Fatal(err) - } - if want := `{"info":{"subdir":"noarch"},"packages":{"keep":{"n":1},"also":{"n":3}},"tail":[]}`; got != want { - t.Errorf("got %s\nwant %s", got, want) - } - if !json.Valid([]byte(got)) { - t.Error("output is not valid JSON") - } -} - -func TestCopyObjectSkipReplaceAndEmpty(t *testing.T) { - got, err := copyString(t, `{"first":1,"secret":"x","n":2,"empty":{}}`, func(m *Member) error { - switch m.Key { - case "first", "secret": - _, err := m.Raw() - return err - case "n": - if _, err := m.Raw(); err != nil { - return err - } - return m.Write([]byte(`3`)) - case "empty": - return m.Object(func(*Member) error { return nil }) - } - return nil - }) - if err != nil { - t.Fatal(err) - } - if want := `{"n":3,"empty":{}}`; got != want { - t.Errorf("got %s, want %s", got, want) - } -} - -func TestCopyObjectKeys(t *testing.T) { - got, err := copyString(t, `{"a 0 && limit < noLimit { - return limit, true - } - } - return 0, false -} - -// ownCgroups returns the process's cgroup v2 path and its cgroup v1 memory -// controller path from /proc/self/cgroup, each "" when not listed. -func ownCgroups(fsys fs.FS) (v2, v1 string) { - f, err := fsys.Open("proc/self/cgroup") - if err != nil { - return "", "" - } - defer func() { _ = f.Close() }() - scanner := bufio.NewScanner(f) - for scanner.Scan() { - // Lines are "hierarchy-ID:controller-list:cgroup-path". - id, rest, ok := strings.Cut(scanner.Text(), ":") - if !ok { - continue - } - controllers, cgroup, ok := strings.Cut(rest, ":") - if !ok { - continue - } - switch { - case id == "0" && controllers == "": - v2 = cgroup - case hasController(controllers, "memory"): - v1 = cgroup - } - } - return v2, v1 -} - -func hasController(list, name string) bool { - for controller := range strings.SplitSeq(list, ",") { - if controller == name { - return true - } - } - return false -} diff --git a/internal/memlimit/memlimit_test.go b/internal/memlimit/memlimit_test.go deleted file mode 100644 index bf8474b..0000000 --- a/internal/memlimit/memlimit_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package memlimit - -import ( - "testing" - "testing/fstest" -) - -func TestCgroupLimit(t *testing.T) { - const gib = 1 << 30 - for _, tt := range []struct { - name string - files map[string]string - want int64 - }{ - { - name: "cgroup v2 container", - files: map[string]string{"proc/self/cgroup": "0::/\n", "sys/fs/cgroup/memory.max": "2147483648\n"}, - want: 2 * gib, - }, - { - name: "cgroup v2 own cgroup", - files: map[string]string{ - "proc/self/cgroup": "0::/system.slice/proxy.service\n", - "sys/fs/cgroup/system.slice/proxy.service/memory.max": "1073741824\n", - }, - want: gib, - }, - { - name: "cgroup v2 unlimited", - files: map[string]string{"proc/self/cgroup": "0::/\n", "sys/fs/cgroup/memory.max": "max\n"}, - }, - { - name: "cgroup v1", - files: map[string]string{ - "proc/self/cgroup": "12:cpu,cpuacct:/\n4:memory:/\n", - "sys/fs/cgroup/memory/memory.limit_in_bytes": "536870912\n", - }, - want: gib / 2, - }, - { - name: "cgroup v1 unlimited", - files: map[string]string{ - "proc/self/cgroup": "4:memory:/\n", - "sys/fs/cgroup/memory/memory.limit_in_bytes": "9223372036854771712\n", - }, - }, - { - name: "malformed", - files: map[string]string{"sys/fs/cgroup/memory.max": "lots\n"}, - }, - { - name: "no cgroup files", - }, - } { - t.Run(tt.name, func(t *testing.T) { - fsys := fstest.MapFS{} - for name, data := range tt.files { - fsys[name] = &fstest.MapFile{Data: []byte(data)} - } - got, ok := cgroupLimit(fsys) - if got != tt.want || ok != (tt.want != 0) { - t.Errorf("cgroupLimit = %d, %v; want %d", got, ok, tt.want) - } - }) - } -} diff --git a/internal/metadata/npm/denylist.go b/internal/metadata/npm/denylist.go deleted file mode 100644 index d34d4d3..0000000 --- a/internal/metadata/npm/denylist.go +++ /dev/null @@ -1,46 +0,0 @@ -package npm - -import "github.com/Masterminds/semver/v3" - -func (h *transformer) applyDenylistFiltering(metadata, versions map[string]any) { - denied := h.DeniedVersions - if len(denied) == 0 { - return - } - times, _ := metadata["time"].(map[string]any) - for version := range denied { - delete(versions, version) - delete(times, version) - } - tags, _ := metadata["dist-tags"].(map[string]any) - for tag, value := range tags { - version, _ := value.(string) - if !denied[version] { - continue - } - delete(tags, tag) - // Custom tags describe publisher intent; do not retarget them. For - // latest, use the highest remaining stable version, even in abbreviated - // packuments without publication timestamps. - if tag == "latest" { - if latest := newestStableNPMVersion(versions); latest != "" { - tags[tag] = latest - } - } - } -} - -func newestStableNPMVersion(versions map[string]any) string { - var best *semver.Version - result := "" - for version := range versions { - parsed, err := semver.StrictNewVersion(version) - if err != nil || parsed.Prerelease() != "" { - continue - } - if best == nil || parsed.GreaterThan(best) || (parsed.Equal(best) && version > result) { - best, result = parsed, version - } - } - return result -} diff --git a/internal/metadata/npm/imports_test.go b/internal/metadata/npm/imports_test.go deleted file mode 100644 index 19f828c..0000000 --- a/internal/metadata/npm/imports_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package npm - -import ( - "os/exec" - "slices" - "strings" - "testing" -) - -// TestPortableImports keeps the package free of the proxy's infrastructure: -// no handler, database, storage, metrics, HTTP or logging, so it builds for -// targets such as WASI where those are not available. -func TestPortableImports(t *testing.T) { - goTool, err := exec.LookPath("go") - if err != nil { - t.Skip("go tool not found") - } - out, err := exec.Command(goTool, "list", "-deps", "-f", "{{.Standard}} {{.ImportPath}}", ".").Output() - if err != nil { - t.Fatalf("go list: %v", err) - } - allowed := []string{ - "github.com/Masterminds/semver/v3", - "github.com/git-pkgs/cooldown", - "github.com/git-pkgs/proxy/internal/jsonscan", - "github.com/git-pkgs/proxy/internal/jsonstream", - "github.com/git-pkgs/proxy/internal/metadata/npm", - "github.com/git-pkgs/proxy/internal/packageurl", - "github.com/git-pkgs/purl", - "github.com/git-pkgs/vers", - "github.com/package-url/packageurl-go", - } - // database/sql/driver comes in through semver, which implements its - // Valuer interface; it is only interfaces, no driver. - forbiddenStd := []string{"database/sql", "log/slog", "net", "net/http"} - for line := range strings.Lines(string(out)) { - standard, path, _ := strings.Cut(strings.TrimSpace(line), " ") - if standard == "true" { - if slices.Contains(forbiddenStd, path) { - t.Errorf("imports %s", path) - } - continue - } - if !slices.Contains(allowed, path) { - t.Errorf("imports %s, which is not on the portable list", path) - } - } -} diff --git a/internal/metadata/npm/metadata.go b/internal/metadata/npm/metadata.go deleted file mode 100644 index 7fc4a40..0000000 --- a/internal/metadata/npm/metadata.go +++ /dev/null @@ -1,551 +0,0 @@ -// Package npm transforms registry metadata without storage or network access. -package npm - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "maps" - "net/url" - "reflect" - "slices" - "sort" - "strings" - "time" - - "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/proxy/internal/jsonscan" - "github.com/git-pkgs/proxy/internal/packageurl" -) - -var ErrVersionDenied = errors.New("package version blocked by denylist") -var ErrVersionNotFound = errors.New("version not found") - -// CooldownPolicy evaluates version age at the caller's supplied time. -type CooldownPolicy interface { - Enabled() bool - Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision -} - -// LocalVersion is an already approved hosted version and its publication time. -type LocalVersion struct { - Version string - Entry json.RawMessage - PublishedAt string -} - -// Options supplies the public URL, policy and hosted versions for a rewrite. -// EvaluatedAt is used as given; the transformation never reads the clock. -type Options struct { - ProxyURL string - Cooldown CooldownPolicy - DeniedVersions map[string]bool - EvaluatedAt time.Time - LocalVersions []LocalVersion - // Optional callbacks report changes synchronously without retaining events. - // OnTarballsRewritten is called at most once per document, with how many - // tarball URLs it pointed at the proxy, and not at all when none. - OnTarballsRewritten func(count int) - OnCooldownFilter func(version, publishedAt string) -} - -type transformer struct{ Options } - -// Rewrite filters upstream versions and points their tarball and attestation -// URLs at the proxy, preserving untouched JSON values verbatim. Hosted entries -// override upstream versions. The returned version set is unordered. The -// document, denylist and hosted entries are not modified. -func Rewrite(packageName string, body []byte, options Options) ([]byte, []string, error) { - return (&transformer{options}).rewriteMetadataWithLocal(packageName, body, options.LocalVersions) -} - -// Version resolves a version or dist-tag through the filtered packument. -func Version(packageName, specifier string, packument []byte, options Options) ([]byte, error) { - filtered, _, err := Rewrite(packageName, packument, options) - if err != nil { - return nil, err - } - - target := specifier - if resolved, ok, err := jsonscan.LookupString(filtered, "dist-tags", specifier); err != nil { - return nil, err - } else if ok { - target = resolved - } - - if options.DeniedVersions[target] { - return nil, ErrVersionDenied - } - - versionRaw, err := jsonscan.Lookup(filtered, "versions", target) - if err != nil { - return nil, err - } - if len(versionRaw) == 0 { - return nil, ErrVersionNotFound - } - - var out bytes.Buffer - (&transformer{options}).writeNPMVersion(&out, packageName, target, versionRaw) - return out.Bytes(), nil -} - -// Abbreviated reports whether body is an abbreviated packument: one with the -// top-level "modified" field the abbreviated format carries and no "time" -// map, which cooldown needs. A full document without a time map, as some -// registries serve, is not abbreviated, and neither is anything that is not -// a JSON object. -func Abbreviated(body []byte) bool { - var hasTime, hasModified bool - err := jsonscan.EachMember(body, func(m jsonscan.Member) error { - switch key := m.Key(body); { - case jsonscan.KeyIs(key, "time"): - hasTime = true - case jsonscan.KeyIs(key, "modified"): - hasModified = true - } - return nil - }) - return err == nil && hasModified && !hasTime -} - -func (h *transformer) rewriteMetadataWithLocal(packageName string, body []byte, local []LocalVersion) ([]byte, []string, error) { - doc, err := indexNPMPackument(body) - if err != nil { - return nil, nil, err - } - - versionsRaw := doc.value(doc.versionsAt) - if len(versionsRaw) == 0 || versionsRaw[0] != '{' { - version, ok, err := jsonscan.LookupString(body, "version") - if err != nil { - return nil, nil, err - } - if ok { - if h.DeniedVersions[version] { - return nil, nil, ErrVersionDenied - } - var out bytes.Buffer - if h.writeNPMVersion(&out, packageName, version, body) { - h.tarballsRewritten(1) - } - return out.Bytes(), []string{version}, nil - } - if len(h.DeniedVersions) != 0 { - return nil, nil, errors.New("npm metadata has no versions object") - } - return body, nil, nil // No versions to rewrite - } - - // The filters only look at which versions exist, so they run on a map of - // version names alongside the decoded time map and dist-tags. - versions := map[string]any{} - if err := jsonscan.EachMember(versionsRaw, func(m jsonscan.Member) error { - version, err := jsonscan.Key(m.Key(versionsRaw)) - versions[version] = nil - return err - }); err != nil { - return nil, nil, err - } - metadata := map[string]any{} - timeMap := doc.decodeObject(doc.timeAt, metadata, "time") - distTags := doc.decodeObject(doc.tagsAt, metadata, "dist-tags") - timeEntries := len(timeMap) - originalTags := maps.Clone(distTags) - - h.applyCooldownFiltering(metadata, versions, packageName) - h.applyDenylistFiltering(metadata, versions) - for _, e := range local { - delete(versions, e.Version) - } - - // Untouched members are copied as they are; the three the filters - // handled are written from their filtered state, and only when it changed. - out, err := doc.write(func(out *bytes.Buffer, i int) error { - switch { - case i == doc.versionsAt: - return h.writeNPMVersions(out, packageName, versionsRaw, versions, local) - case i == doc.timeAt && timeMap != nil && (len(timeMap) != timeEntries || len(local) > 0): - pushedAt := make([][2]string, 0, len(local)) - for _, e := range local { - pushedAt = append(pushedAt, [2]string{e.Version, e.PublishedAt}) - } - return jsonscan.WriteFilteredObject(out, doc.value(i), func(key string) bool { - _, ok := timeMap[key] - pushed := slices.ContainsFunc(local, func(e LocalVersion) bool { return e.Version == key }) - return ok && !pushed - }, pushedAt) - case i == doc.tagsAt && distTags != nil && !reflect.DeepEqual(distTags, originalTags): - encoded, err := json.Marshal(distTags) - out.Write(encoded) - return err - default: - out.Write(doc.value(i)) - return nil - } - }) - kept := slices.Collect(maps.Keys(versions)) - for _, e := range local { - kept = append(kept, e.Version) - } - return out, kept, err -} - -// npmPackument is a packument's top-level members as offsets into its bytes, -// with the positions of the members rewriteMetadata filters, or -1 for those -// it lacks. A key that repeats is recorded at its last occurrence, the one -// JSON.parse keeps. -type npmPackument struct { - body []byte - members []jsonscan.Member - versionsAt, timeAt, tagsAt int -} - -func indexNPMPackument(body []byte) (*npmPackument, error) { - if !json.Valid(body) { - return nil, errors.New("npm metadata is not valid JSON") - } - doc := &npmPackument{body: body, versionsAt: -1, timeAt: -1, tagsAt: -1} - err := jsonscan.EachMember(body, func(m jsonscan.Member) error { - switch key := m.Key(body); { - case jsonscan.KeyIs(key, "versions"): - doc.versionsAt = len(doc.members) - case jsonscan.KeyIs(key, "time"): - doc.timeAt = len(doc.members) - case jsonscan.KeyIs(key, "dist-tags"): - doc.tagsAt = len(doc.members) - } - doc.members = append(doc.members, m) - return nil - }) - return doc, err -} - -// value returns the raw value of member i, or nil when i is -1. -func (d *npmPackument) value(i int) []byte { - if i < 0 { - return nil - } - return d.members[i].Value(d.body) -} - -// decodeObject decodes member i into metadata[name] and returns it when it is -// an object. The filters edit the returned map in place. -func (d *npmPackument) decodeObject(i int, metadata map[string]any, name string) map[string]any { - raw := d.value(i) - if raw == nil { - return nil - } - var value any - if err := json.Unmarshal(raw, &value); err != nil { - return nil - } - metadata[name] = value - object, _ := value.(map[string]any) - return object -} - -// write assembles the document in member order, with writeValue writing each -// member's value. Earlier duplicates of the filtered keys are dropped: they -// would carry unfiltered data, and JSON.parse ignores them anyway. -func (d *npmPackument) write(writeValue func(out *bytes.Buffer, i int) error) ([]byte, error) { - var out bytes.Buffer - out.Grow(len(d.body) + len(d.body)/8) - out.WriteByte('{') - first := true - for i, m := range d.members { - if i != d.versionsAt && i != d.timeAt && i != d.tagsAt { - key := m.Key(d.body) - if jsonscan.KeyIs(key, "versions") || jsonscan.KeyIs(key, "time") || jsonscan.KeyIs(key, "dist-tags") { - continue - } - } - if !first { - out.WriteByte(',') - } - first = false - out.Write(m.Key(d.body)) - out.WriteByte(':') - if err := writeValue(&out, i); err != nil { - return nil, err - } - } - out.WriteByte('}') - return out.Bytes(), nil -} - -// writeNPMVersions writes the versions object, keeping the versions still in -// keep and pointing each kept version's tarball at this proxy, followed by the -// pushed versions in local. -func (h *transformer) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any, local []LocalVersion) error { - out.WriteByte('{') - first := true - rewritten := 0 - err := jsonscan.EachMember(versionsRaw, func(m jsonscan.Member) error { - version, err := jsonscan.Key(m.Key(versionsRaw)) - if err != nil { - return err - } - if _, ok := keep[version]; !ok { - return nil - } - if !first { - out.WriteByte(',') - } - first = false - out.Write(m.Key(versionsRaw)) - out.WriteByte(':') - if h.writeNPMVersion(out, packageName, version, m.Value(versionsRaw)) { - rewritten++ - } - return nil - }) - h.tarballsRewritten(rewritten) - for _, e := range local { - if !first { - out.WriteByte(',') - } - first = false - key, _ := json.Marshal(e.Version) - out.Write(key) - out.WriteByte(':') - out.Write(e.Entry) - } - out.WriteByte('}') - return err -} - -// writeNPMVersion writes one version entry with its dist.tarball and -// dist.attestations.url pointing at this proxy, and reports whether it -// rewrote the tarball. Members that are missing or not strings are written -// unchanged. -func (h *transformer) writeNPMVersion(out *bytes.Buffer, packageName, version string, raw []byte) bool { - dist, ok, err := jsonscan.FindMember(raw, "dist") - if err != nil || !ok { - out.Write(raw) - return false - } - distRaw := dist.Value(raw) - - var buf [2]urlEdit - edits := buf[:0] - tarball, ok := stringMember(distRaw, "tarball") - if ok { - tarball.replacement = TarballURL(h.ProxyURL, packageName, version, tarball.old) - edits = append(edits, tarball.shift(dist.ValStart)) - } - if attestations, found, err := jsonscan.FindMember(distRaw, "attestations"); err == nil && found { - if att, ok := stringMember(attestations.Value(distRaw), "url"); ok { - att.replacement = AttestationsURL(h.ProxyURL, packageName, version) - edits = append(edits, att.shift(dist.ValStart+attestations.ValStart)) - } - } - writeEdits(out, raw, edits) - return tarball.replacement != "" -} - -// urlEdit replaces the string value at raw[start:end], which decodes to old, -// with replacement. -type urlEdit struct { - start, end int - old string - replacement string -} - -func (e urlEdit) shift(offset int) urlEdit { - e.start += offset - e.end += offset - return e -} - -// stringMember finds the string member name in obj. -func stringMember(obj []byte, name string) (urlEdit, bool) { - m, ok, err := jsonscan.FindMember(obj, name) - if err != nil || !ok || obj[m.ValStart] != '"' { - return urlEdit{}, false - } - var old string - if err := json.Unmarshal(m.Value(obj), &old); err != nil { - return urlEdit{}, false - } - return urlEdit{start: m.ValStart, end: m.ValEnd, old: old}, true -} - -// writeEdits writes raw with the edits applied. There are at most two, which -// may come in either order. -func writeEdits(out *bytes.Buffer, raw []byte, edits []urlEdit) { - if len(edits) == 2 && edits[1].start < edits[0].start { - edits[0], edits[1] = edits[1], edits[0] - } - pos := 0 - for _, e := range edits { - encoded, err := json.Marshal(e.replacement) - if err != nil { - continue - } - out.Write(raw[pos:e.start]) - out.Write(encoded) - pos = e.end - } - out.Write(raw[pos:]) -} - -// tarballsRewritten reports once per document how many tarball URLs it -// rewrote. A report per version flooded debug logs: packages such as eslint -// list hundreds of versions, and with cooldown on every request rewrites the -// document again. -func (h *transformer) tarballsRewritten(count int) { - if count > 0 && h.OnTarballsRewritten != nil { - h.OnTarballsRewritten(count) - } -} - -// applyCooldownFiltering removes versions that are too recently published, -// and updates the dist-tags that named them. -func (h *transformer) applyCooldownFiltering(metadata map[string]any, versions map[string]any, packageName string) { - if h.Cooldown == nil || !h.Cooldown.Enabled() { - return - } - - timeMap, _ := metadata["time"].(map[string]any) - if timeMap == nil { - return - } - - packagePURL := packageurl.MakeString("npm", packageName, "") - - withheld := map[string]bool{} - for version := range versions { - publishedStr, ok := timeMap[version].(string) - if !ok { - continue - } - publishedAt, err := time.Parse(time.RFC3339, publishedStr) - if err != nil { - continue - } - if !h.Cooldown.Evaluate("npm", packagePURL, publishedAt, h.EvaluatedAt).Allowed { - if h.OnCooldownFilter != nil { - h.OnCooldownFilter(version, publishedStr) - } - delete(versions, version) - delete(timeMap, version) - withheld[version] = true - } - } - - h.updateDistTags(metadata, versions, timeMap, withheld) -} - -// updateDistTags moves dist-tags.latest to the newest remaining version if -// the current latest was removed by cooldown filtering, and, as the denylist -// does, drops other tags that name a withheld version. -func (h *transformer) updateDistTags(metadata, versions, timeMap map[string]any, withheld map[string]bool) { - distTags, ok := metadata["dist-tags"].(map[string]any) - if !ok { - return - } - - for tag, value := range distTags { - version, _ := value.(string) - if tag != "latest" && withheld[version] { - delete(distTags, tag) - } - } - - latest, ok := distTags["latest"].(string) - if !ok { - return - } - - if _, exists := versions[latest]; exists { - return - } - - if newLatest := h.findNewestVersion(versions, timeMap); newLatest != "" { - distTags["latest"] = newLatest - } else if withheld[latest] { - delete(distTags, "latest") - } -} - -// TarballURL returns the proxy URL that serves the tarball upstream -// lists at tarball for this version. -func TarballURL(proxyURL, packageName, version, tarball string) string { - filename := tarball - if idx := strings.LastIndex(tarball, "/"); idx >= 0 { - filename = tarball[idx+1:] - } - if VersionFromFilename(packageName, filename) != version { - _, shortName, scoped := strings.Cut(packageName, "/") - if !scoped { - shortName = packageName - } - filename = shortName + "-" + version + ".tgz" - } - - return fmt.Sprintf("%s/npm/%s/-/%s", proxyURL, url.PathEscape(packageName), filename) -} - -// AttestationsURL returns the proxy URL that serves the attestations of a -// version, in the form the npm registry uses: the package name with its scope -// slash encoded, "@", and the version. -func AttestationsURL(proxyURL, packageName, version string) string { - return fmt.Sprintf("%s/npm/-/npm/v1/attestations/%s@%s", proxyURL, url.PathEscape(packageName), url.PathEscape(version)) -} - -// findNewestVersion returns the version string with the most recent timestamp -// from the remaining versions, using the time map. -func (h *transformer) findNewestVersion(versions map[string]any, timeMap map[string]any) string { - if timeMap == nil { - return "" - } - - type versionTime struct { - version string - t time.Time - } - - var vts []versionTime - for v := range versions { - if ts, ok := timeMap[v].(string); ok { - if t, err := time.Parse(time.RFC3339, ts); err == nil { - vts = append(vts, versionTime{v, t}) - } - } - } - - if len(vts) == 0 { - return "" - } - - sort.Slice(vts, func(i, j int) bool { - return vts[i].t.After(vts[j].t) - }) - - return vts[0].version -} - -// VersionFromFilename extracts a version from an npm tarball filename. -func VersionFromFilename(packageName, filename string) string { - // Remove .tgz extension - if !strings.HasSuffix(filename, ".tgz") { - return "" - } - base := strings.TrimSuffix(filename, ".tgz") - - // For scoped packages, the filename uses the short name - shortName := packageName - if _, name, scoped := strings.Cut(packageName, "/"); scoped { - shortName = name - } - - // Expected format: {shortName}-{version} - prefix := shortName + "-" - if !strings.HasPrefix(base, prefix) { - return "" - } - - return strings.TrimPrefix(base, prefix) -} diff --git a/internal/metadata/npm/metadata_test.go b/internal/metadata/npm/metadata_test.go deleted file mode 100644 index 2748ed3..0000000 --- a/internal/metadata/npm/metadata_test.go +++ /dev/null @@ -1,232 +0,0 @@ -package npm - -import ( - "encoding/json" - "fmt" - "github.com/git-pkgs/cooldown" - "strings" - "testing" - "time" -) - -// TestNPMRewriteMetadataKeepsUpstreamBytes checks that only tarball URLs -// change: everything else, including numbers encoding/json would round -// through float64 and characters it would HTML-escape, is copied verbatim. -func TestNPMRewriteMetadataKeepsUpstreamBytes(t *testing.T) { - options := Options{ProxyURL: "http://proxy"} - input := `{"name":"demo","_big":12345678901234567890,"readme":"&",` + - `"versions":{"1.0.0":{"version":"1.0.0","dist":{"shasum":"s","tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz","integrity":"i"}}}}` - - out, _, err := Rewrite("demo", []byte(input), options) - if err != nil { - t.Fatal(err) - } - want := strings.Replace(input, "https://registry.npmjs.org/demo/-/demo-1.0.0.tgz", "http://proxy/npm/demo/-/demo-1.0.0.tgz", 1) - if string(out) != want { - t.Errorf("rewrite =\n%s\nwant\n%s", out, want) - } -} - -func TestNPMRewriteMetadataUnusualEntries(t *testing.T) { - options := Options{ProxyURL: "http://proxy"} - input := `{ - "versions": { - "1.0.0": "not an object", - "2.0.0": {"dist": "no object either"}, - "3.0.0": {"dist": {"tarball": 7}}, - "4.0.0": {"dist": {"tarball": "https://registry.npmjs.org/demo/-/renamed.tgz"}}, - "5.0.0": {"dist": {}} - } - }` - out, _, err := Rewrite("demo", []byte(input), options) - if err != nil { - t.Fatal(err) - } - var doc struct { - Versions map[string]json.RawMessage `json:"versions"` - } - if err := json.Unmarshal(out, &doc); err != nil { - t.Fatalf("output is not JSON: %v\n%s", err, out) - } - for v, want := range map[string]string{ - "1.0.0": `"not an object"`, - "2.0.0": `{"dist": "no object either"}`, - "3.0.0": `{"dist": {"tarball": 7}}`, - "4.0.0": `{"dist": {"tarball": "http://proxy/npm/demo/-/demo-4.0.0.tgz"}}`, - "5.0.0": `{"dist": {}}`, - } { - if got := string(doc.Versions[v]); got != want { - t.Errorf("%s = %s, want %s", v, got, want) - } - } -} - -func TestNPMRewriteMetadataInvalidJSON(t *testing.T) { - options := Options{ProxyURL: "http://proxy"} - for _, input := range []string{`{"versions": {}`, `[1]`, `{"versions": {}} trailing`} { - if _, _, err := Rewrite("demo", []byte(input), options); err == nil { - t.Errorf("%s: expected an error", input) - } - } -} - -func TestNPMRewriteMetadataNoVersions(t *testing.T) { - options := Options{ProxyURL: "http://proxy"} - input := `{"name":"demo","versions":[]}` - out, _, err := Rewrite("demo", []byte(input), options) - if err != nil || string(out) != input { - t.Errorf("rewrite = %s, %v; want input unchanged", out, err) - } - - options.DeniedVersions = map[string]bool{"1.0.0": true} - if _, _, err := Rewrite("demo", []byte(input), options); err == nil { - t.Error("expected an error when a denylisted package has no versions object") - } -} - -// TestNPMRewriteMetadataCooldownAndDenylist runs both filters together and -// checks that versions, their time entries and the tags pointing at them are -// all removed, with latest moved to the newest version left. -func TestNPMRewriteMetadataCooldownAndDenylist(t *testing.T) { - now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) - ts := func(age time.Duration) string { return now.Add(-age).UTC().Format(time.RFC3339) } - - options := Options{ProxyURL: "http://proxy", EvaluatedAt: time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)} - options.Cooldown = &cooldown.Config{Default: "3d"} - options.DeniedVersions = map[string]bool{"2.0.0": true} - - version := func(v string) string { - return fmt.Sprintf(`%q:{"version":%q,"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-%s.tgz"}}`, v, v, v) - } - input := `{"name":"demo",` + - `"dist-tags":{"latest":"3.0.0","beta":"2.0.0","next":"1.0.0"},` + - `"versions":{` + version("1.0.0") + `,` + version("2.0.0") + `,` + version("3.0.0") + `},` + - `"time":{"created":"` + ts(1000*time.Hour) + `","1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}}` - - out, _, err := Rewrite("demo", []byte(input), options) - if err != nil { - t.Fatal(err) - } - var doc struct { - DistTags map[string]string `json:"dist-tags"` - Versions map[string]json.RawMessage `json:"versions"` - Time map[string]string `json:"time"` - } - if err := json.Unmarshal(out, &doc); err != nil { - t.Fatalf("output is not JSON: %v\n%s", err, out) - } - if len(doc.Versions) != 1 || doc.Versions["1.0.0"] == nil { - t.Errorf("versions = %v, want only 1.0.0", doc.Versions) - } - if _, ok := doc.Time["2.0.0"]; ok { - t.Error("time still lists denylisted 2.0.0") - } - if _, ok := doc.Time["3.0.0"]; ok { - t.Error("time still lists 3.0.0, which is inside cooldown") - } - if _, ok := doc.Time["created"]; !ok { - t.Error("time lost its created entry") - } - want := map[string]string{"latest": "1.0.0", "next": "1.0.0"} - if fmt.Sprint(doc.DistTags) != fmt.Sprint(want) { - t.Errorf("dist-tags = %v, want %v", doc.DistTags, want) - } - if !strings.Contains(string(doc.Versions["1.0.0"]), "http://proxy/npm/demo/-/demo-1.0.0.tgz") { - t.Errorf("1.0.0 tarball not rewritten: %s", doc.Versions["1.0.0"]) - } -} - -// TestNPMRewriteMetadataDuplicateKeys checks that a repeated top-level key -// cannot smuggle an unfiltered copy of the versions past the filters. -func TestNPMRewriteMetadataDuplicateKeys(t *testing.T) { - options := Options{ProxyURL: "http://proxy", EvaluatedAt: time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)} - options.DeniedVersions = map[string]bool{"2.0.0": true} - input := `{"versions":{"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}},` + - `"versions":{"1.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"}},"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}}}` - out, _, err := Rewrite("demo", []byte(input), options) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(out), "2.0.0") || strings.Count(string(out), `"versions"`) != 1 { - t.Errorf("rewrite = %s", out) - } -} - -func TestAbbreviated(t *testing.T) { - for body, want := range map[string]bool{ - `{"name":"demo","modified":"2024-01-01T00:00:00Z","versions":{}}`: true, - `{"name":"demo","modified":"2024-01-01T00:00:00Z","time":{},"versions":{}}`: false, - `{"name":"demo","versions":{}}`: false, - `["modified"]`: false, - `{"modified":`: false, - } { - if got := Abbreviated([]byte(body)); got != want { - t.Errorf("Abbreviated(%s) = %v, want %v", body, got, want) - } - } -} - -// TestNPMRewriteAttestationsURL checks that dist.attestations.url points at -// the proxy whichever order dist lists its members in, and that nothing else -// in dist changes. -func TestNPMRewriteAttestationsURL(t *testing.T) { - options := Options{ProxyURL: "http://proxy"} - tests := []struct{ name, version, dist, want string }{ - {"demo", "1.0.0", - `{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz","attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/demo@1.0.0","provenance":{"predicateType":"p"}}}`, - `{"tarball":"http://proxy/npm/demo/-/demo-1.0.0.tgz","attestations":{"url":"http://proxy/npm/-/npm/v1/attestations/demo@1.0.0","provenance":{"predicateType":"p"}}}`}, - {"@scope/demo", "2.0.0-rc.1", - `{"attestations":{"provenance":{},"url":"https://registry.npmjs.org/-/npm/v1/attestations/@scope%2fdemo@2.0.0-rc.1"},"tarball":"https://registry.npmjs.org/@scope/demo/-/demo-2.0.0-rc.1.tgz"}`, - `{"attestations":{"provenance":{},"url":"http://proxy/npm/-/npm/v1/attestations/@scope%2Fdemo@2.0.0-rc.1"},"tarball":"http://proxy/npm/@scope%2Fdemo/-/demo-2.0.0-rc.1.tgz"}`}, - {"demo", "3.0.0", - `{"attestations":{"url":7}}`, - `{"attestations":{"url":7}}`}, - } - for _, tt := range tests { - input := `{"versions":{"` + tt.version + `":{"dist":` + tt.dist + `}}}` - out, _, err := Rewrite(tt.name, []byte(input), options) - if err != nil { - t.Fatal(err) - } - want := `{"versions":{"` + tt.version + `":{"dist":` + tt.want + `}}}` - if string(out) != want { - t.Errorf("%s@%s: rewrite =\n%s\nwant\n%s", tt.name, tt.version, out, want) - } - } -} - -// TestNPMRewriteCooldownDropsTags checks that cooldown, like the denylist, -// drops a tag other than latest that names a version it withholds, in both -// the in-memory and the streamed rewrite. -func TestNPMRewriteCooldownDropsTags(t *testing.T) { - now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) - ts := func(age time.Duration) string { return now.Add(-age).UTC().Format(time.RFC3339) } - options := Options{ProxyURL: "http://proxy", EvaluatedAt: now, Cooldown: &cooldown.Config{Default: "3d"}} - input := `{"name":"demo",` + - `"dist-tags":{"latest":"3.0.0","next":"3.0.0","legacy":"1.0.0"},` + - `"versions":{"1.0.0":{},"2.0.0":{},"3.0.0":{}},` + - `"time":{"1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}}` - - rewrite := map[string]func() ([]byte, []string, error){ - "memory": func() ([]byte, []string, error) { return Rewrite("demo", []byte(input), options) }, - "stream": func() ([]byte, []string, error) { return rewriteStream("demo", []byte(input), options) }, - } - for name, run := range rewrite { - t.Run(name, func(t *testing.T) { - out, _, err := run() - if err != nil { - t.Fatal(err) - } - var doc struct { - DistTags map[string]string `json:"dist-tags"` - } - if err := json.Unmarshal(out, &doc); err != nil { - t.Fatalf("output is not JSON: %v\n%s", err, out) - } - want := map[string]string{"latest": "2.0.0", "legacy": "1.0.0"} - if fmt.Sprint(doc.DistTags) != fmt.Sprint(want) { - t.Errorf("dist-tags = %v, want %v", doc.DistTags, want) - } - }) - } -} diff --git a/internal/metadata/npm/npmbench/npmbench.go b/internal/metadata/npm/npmbench/npmbench.go deleted file mode 100644 index cc4ad57..0000000 --- a/internal/metadata/npm/npmbench/npmbench.go +++ /dev/null @@ -1,42 +0,0 @@ -// Package npmbench builds npm packuments for benchmarks. -package npmbench - -import ( - "fmt" - "strings" -) - -// readmeSentences makes each version's readme about 4KB. -const readmeSentences = 160 - -// Packument builds a full packument shaped like a large real one: versions -// 1.0.0 to 1.0.{versions-1}, each with dependencies, scripts and a readme, -// and a time entry for each. -func Packument(versions int) []byte { - var b strings.Builder - readme := strings.Repeat("Some documentation text. ", readmeSentences) - b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"1.0.` + fmt.Sprint(versions-1) + `"},"versions":{`) - for i := range versions { - if i > 0 { - b.WriteByte(',') - } - v := fmt.Sprintf("1.0.%d", i) - fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"description":"A big package","main":"index.js",`+ - `"scripts":{"test":"node test.js","build":"tsc -p ."},"dependencies":{`, v, v) - for d := range 25 { - if d > 0 { - b.WriteByte(',') - } - fmt.Fprintf(&b, `"dep-%d":"^%d.0.0"`, d, d) - } - fmt.Fprintf(&b, `},"readme":%q,"dist":{"shasum":"0123456789abcdef0123456789abcdef01234567",`+ - `"integrity":"sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",`+ - `"tarball":"https://registry.npmjs.org/big/-/big-%s.tgz"}}`, readme, v) - } - b.WriteString(`},"time":{"created":"2015-01-01T00:00:00.000Z"`) - for i := range versions { - fmt.Fprintf(&b, `,"1.0.%d":"2016-01-01T00:00:00.000Z"`, i) - } - b.WriteString(`}}`) - return []byte(b.String()) -} diff --git a/internal/metadata/npm/policy_test.go b/internal/metadata/npm/policy_test.go deleted file mode 100644 index 7726d29..0000000 --- a/internal/metadata/npm/policy_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package npm - -import ( - "bytes" - _ "embed" - "encoding/json" - "errors" - "slices" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -//go:embed testdata/packument.json -var packument []byte - -func TestRewriteCallbacks(t *testing.T) { - options := Options{ProxyURL: "https://proxy.example", Cooldown: &cooldown.Config{Default: "7d"}, - DeniedVersions: map[string]bool{"2.0.0": true}, EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)} - want, kept, err := Rewrite("@example/widgets", packument, options) - if err != nil { - t.Fatal(err) - } - var rewrites, filtered int - options.OnTarballsRewritten = func(count int) { - rewrites++ - if count != 1 { - t.Errorf("rewrite count = %d, want 1", count) - } - } - options.OnCooldownFilter = func(version, publishedAt string) { - filtered++ - if version != "3.0.0" || publishedAt != "2099-01-01T00:00:00Z" { - t.Errorf("filter event: %s, %s", version, publishedAt) - } - } - out, gotKept, err := Rewrite("@example/widgets", packument, options) - if err != nil || !bytes.Equal(out, want) || !slices.Equal(gotKept, kept) || rewrites != 1 || filtered != 1 { - t.Fatalf("callbacks: rewrites=%d, filtered=%d, kept=%v, error=%v", rewrites, filtered, gotKept, err) - } -} - -func TestRewriteHostedOverride(t *testing.T) { - options := Options{ - ProxyURL: "https://proxy.example", - Cooldown: &cooldown.Config{Default: "7d"}, - DeniedVersions: map[string]bool{"2.0.0": true}, - EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC), - LocalVersions: []LocalVersion{{Version: "3.0.0", PublishedAt: "2030-01-01T00:00:00Z", - Entry: json.RawMessage(`{"version":"3.0.0","dist":{"tarball":"https://proxy.example/npm/@example%2Fwidgets/-/widgets-3.0.0.tgz","integrity":"sha512-local"}}`)}}, - } - before := bytes.Clone(packument) - localBefore := bytes.Clone(options.LocalVersions[0].Entry) - out, kept, err := Rewrite("@example/widgets", packument, options) - if err != nil { - t.Fatal(err) - } - slices.Sort(kept) - if !slices.Equal(kept, []string{"1.0.0", "3.0.0"}) { - t.Fatalf("kept = %v", kept) - } - var doc struct { - Versions map[string]json.RawMessage `json:"versions"` - Tags map[string]string `json:"dist-tags"` - Time map[string]string `json:"time"` - } - if err := json.Unmarshal(out, &doc); err != nil { - t.Fatal(err) - } - if len(doc.Versions) != 2 || !bytes.Equal(doc.Versions["3.0.0"], localBefore) || doc.Time["3.0.0"] != options.LocalVersions[0].PublishedAt { - t.Fatalf("hosted override lost: %s", out) - } - if len(doc.Tags) != 1 || doc.Tags["latest"] != "1.0.0" || doc.Time["2.0.0"] != "" || doc.Time["created"] == "" { - t.Fatalf("policy output: %s", out) - } - for _, want := range []string{"12345678901234567890", "Widgets & tools", "https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz"} { - if !strings.Contains(string(out), want) { - t.Errorf("output lost %q", want) - } - } - if !bytes.Equal(packument, before) || !bytes.Equal(options.LocalVersions[0].Entry, localBefore) || len(options.DeniedVersions) != 1 || !options.DeniedVersions["2.0.0"] { - t.Fatal("rewrite mutated its input") - } - again, _, err := Rewrite("@example/widgets", packument, options) - if err != nil || !bytes.Equal(out, again) { - t.Fatalf("repeat changed output: %v", err) - } -} - -func TestVersionPolicyErrors(t *testing.T) { - options := Options{ProxyURL: "https://proxy.example", Cooldown: &cooldown.Config{Default: "7d"}, - DeniedVersions: map[string]bool{"2.0.0": true}, EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)} - for _, tc := range []struct { - specifier string - want error - }{ - {"1.0.0", nil}, {"latest", nil}, {"2.0.0", ErrVersionDenied}, {"3.0.0", ErrVersionNotFound}, {"beta", ErrVersionNotFound}, {"missing", ErrVersionNotFound}, - } { - t.Run(tc.specifier, func(t *testing.T) { - var counts []int - options := options - options.OnTarballsRewritten = func(count int) { counts = append(counts, count) } - out, err := Version("@example/widgets", tc.specifier, packument, options) - if !errors.Is(err, tc.want) { - t.Fatalf("error = %v, want %v", err, tc.want) - } - // The packument is reported once; picking one version out of it - // is not a second report. - if !slices.Equal(counts, []int{1}) { - t.Errorf("rewrite reports = %v, want [1]", counts) - } - if tc.want == nil && (!bytes.Contains(out, []byte(`"version": "1.0.0"`)) || !bytes.Contains(out, []byte("https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz"))) { - t.Fatalf("resolved version: %s", out) - } - }) - } -} - -func TestRewriteCooldownEvaluationTime(t *testing.T) { - body := []byte(`{"versions":{"1.0.0":{}},"time":{"1.0.0":"2030-01-01T00:00:00Z"}}`) - boundary := time.Date(2030, 1, 8, 0, 0, 0, 0, time.UTC) - options := Options{Cooldown: &cooldown.Config{Default: "7d"}} - for _, tc := range []struct { - at time.Time - count int - }{ - {boundary.Add(-time.Nanosecond), 0}, {boundary, 1}, {boundary.Add(time.Second), 1}, - } { - options.EvaluatedAt = tc.at - _, kept, err := Rewrite("@example/widgets", body, options) - if err != nil || len(kept) != tc.count { - t.Errorf("at %s: kept %v, error %v", tc.at, kept, err) - } - } - options.Cooldown = &cooldown.Config{Default: "7d", Packages: map[string]string{"pkg:npm/%40example/widgets": "0"}} - options.EvaluatedAt = boundary.Add(-time.Hour) - _, kept, err := Rewrite("@example/widgets", body, options) - if err != nil || len(kept) != 1 { - t.Fatalf("scoped exemption: kept %v, error %v", kept, err) - } -} - -func TestRewriteVersionDocument(t *testing.T) { - body := []byte(`{"name":"@example/widgets","version":"1.0.0","dist":{"tarball":"https://cdn.example/abcdef.tgz"}}`) - options := Options{ProxyURL: "https://proxy.example"} - out, kept, err := Rewrite("@example/widgets", body, options) - if err != nil || !slices.Equal(kept, []string{"1.0.0"}) || !bytes.Contains(out, []byte("https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz")) { - t.Fatalf("rewrite: %s, %v, %v", out, kept, err) - } - var counts []int - options.OnTarballsRewritten = func(count int) { counts = append(counts, count) } - if _, _, err := Rewrite("@example/widgets", body, options); err != nil || !slices.Equal(counts, []int{1}) { - t.Fatalf("version document reported %v, %v", counts, err) - } - options.DeniedVersions = map[string]bool{"1.0.0": true} - if _, _, err := Rewrite("@example/widgets", body, options); !errors.Is(err, ErrVersionDenied) { - t.Fatalf("denied version: %v", err) - } -} diff --git a/internal/metadata/npm/stream.go b/internal/metadata/npm/stream.go deleted file mode 100644 index 8788de1..0000000 --- a/internal/metadata/npm/stream.go +++ /dev/null @@ -1,428 +0,0 @@ -package npm - -import ( - "bytes" - "encoding/json" - "encoding/json/jsontext" - "errors" - "fmt" - "io" - "maps" - "reflect" - "slices" - - "github.com/git-pkgs/proxy/internal/jsonstream" -) - -// ErrNotStreamable is returned by RewriteStream for a document it leaves to -// Rewrite: one without a versions object, such as a single version's -// metadata, or a rewrite that lists pushed versions. -var ErrNotStreamable = errors.New("npm metadata can't be rewritten as a stream") - -// Scan is what a first pass over a packument learns: the version names, and -// the time map and dist-tags the filters work on. It holds no version -// entries, so it stays small whatever the size of the packument. -type Scan struct { - versions []string - versionsObject bool - time any - distTags any - // occurrences counts each filtered key, so the second pass can drop the - // earlier duplicates, as Rewrite does. - occurrences map[string]int - hasModified bool -} - -// Abbreviated reports whether the scanned document is an abbreviated -// packument, as the package function Abbreviated does for a document in -// memory. -func (s *Scan) Abbreviated() bool { - return s.hasModified && s.occurrences["time"] == 0 -} - -// ScanPackument reads a packument once, keeping only what RewriteStream -// needs to decide which versions remain. Version entries are read one at a -// time and dropped. -func ScanPackument(r io.Reader) (*Scan, error) { - s := &Scan{occurrences: map[string]int{}} - dec := newDecoder(r) - if err := expectDelim(dec, '{'); err != nil { - return nil, err - } - for dec.PeekKind() != '}' { - key, err := objectKey(dec) - if err != nil { - return nil, err - } - switch key { - case "versions": - s.occurrences[key]++ - s.versions, s.versionsObject, err = objectKeys(dec) - case "time": - s.occurrences[key]++ - s.time = nil - err = decodeValue(dec, &s.time) - case "dist-tags": - s.occurrences[key]++ - s.distTags = nil - err = decodeValue(dec, &s.distTags) - default: - if key == "modified" { - s.hasModified = true - } - err = dec.SkipValue() - } - if err != nil { - return nil, err - } - } - if err := expectDelim(dec, '}'); err != nil { - return nil, err - } - return s, expectEnd(dec) -} - -// RewriteStream is Rewrite for a packument too large to hold in memory. scan -// is the result of ScanPackument over the same bytes r reads again; the -// caller checks they are the same. The rewritten document is written to w -// with the bytes of untouched values copied as they are, and its output is -// the same as Rewrite's for the same document and options. It returns the -// versions it kept. For a document without a versions object, or options -// listing pushed versions, it returns ErrNotStreamable before writing. -func RewriteStream(packageName string, scan *Scan, r io.Reader, w io.Writer, options Options) ([]string, error) { - if !scan.versionsObject || len(options.LocalVersions) > 0 { - return nil, ErrNotStreamable - } - h := &transformer{options} - f := scan.filter(h, packageName) - sw := &streamWriter{ - transformer: h, - packageName: packageName, - scan: scan, - filtered: f, - seen: map[string]int{}, - } - if err := jsonstream.CopyObject(r, w, sw.member); err != nil { - return nil, err - } - h.tarballsRewritten(sw.rewritten) - return slices.Collect(maps.Keys(f.versions)), nil -} - -// filtered is what cooldown and the denylist leave of a scanned packument. -type filtered struct { - versions map[string]any - timeMap, distTags map[string]any - timeChanged, tagsChanged bool -} - -// filter runs the filters Rewrite runs over what scan collected. The scan is -// left as it is, so it can be filtered again. -func (s *Scan) filter(h *transformer, packageName string) filtered { - versions := make(map[string]any, len(s.versions)) - for _, v := range s.versions { - versions[v] = nil - } - metadata := map[string]any{} - timeMap := s.objectValue(metadata, "time", s.time) - distTags := s.objectValue(metadata, "dist-tags", s.distTags) - timeEntries := len(timeMap) - originalTags := maps.Clone(distTags) - - h.applyCooldownFiltering(metadata, versions, packageName) - h.applyDenylistFiltering(metadata, versions) - - return filtered{ - versions: versions, - timeMap: timeMap, - distTags: distTags, - timeChanged: timeMap != nil && len(timeMap) != timeEntries, - tagsChanged: distTags != nil && !reflect.DeepEqual(distTags, originalTags), - } -} - -// streamWriter writes the second pass of RewriteStream from the filtered -// state the first pass left. -type streamWriter struct { - *transformer - filtered - packageName string - scan *Scan - seen map[string]int - entry bytes.Buffer - rewritten int -} - -// member writes one top-level member. Members the filters don't touch are -// left unconsumed, so CopyObject copies them as they are. -func (s *streamWriter) member(m *jsonstream.Member) error { - total, filtered := s.scan.occurrences[m.Key] - if !filtered { - return nil - } - s.seen[m.Key]++ - if s.seen[m.Key] < total { - return m.Skip() // an earlier duplicate, dropped - } - switch { - case m.Key == "versions": - return m.Object(s.version) - case m.Key == "time" && s.timeChanged: - return m.Object(s.time) - case m.Key == "dist-tags" && s.tagsChanged: - if err := m.Skip(); err != nil { - return err - } - encoded, err := json.Marshal(s.distTags) - if err != nil { - return err - } - return m.Write(encoded) - } - return nil -} - -// version writes a kept version with its tarball pointing at the proxy. -func (s *streamWriter) version(v *jsonstream.Member) error { - if _, keep := s.versions[v.Key]; !keep { - return v.Skip() - } - raw, err := v.Raw() - if err != nil { - return err - } - s.entry.Reset() - if s.writeNPMVersion(&s.entry, s.packageName, v.Key, raw) { - s.rewritten++ - } - return v.Write(s.entry.Bytes()) -} - -// time writes a time entry the filters kept. -func (s *streamWriter) time(t *jsonstream.Member) error { - if _, keep := s.timeMap[t.Key]; keep { - return t.Copy() - } - return t.Skip() -} - -// objectValue records value under name in metadata, as Rewrite's -// decodeObject does, and returns it when it is an object. An object is -// copied, since the filters edit it. -func (s *Scan) objectValue(metadata map[string]any, name string, value any) map[string]any { - if s.occurrences[name] == 0 { - return nil - } - object, ok := value.(map[string]any) - if !ok { - metadata[name] = value - return nil - } - object = maps.Clone(object) - metadata[name] = object - return object -} - -// newDecoder reads JSON as encoding/json does: of repeated names the last -// counts, and invalid UTF-8 in strings is replaced rather than refused. -func newDecoder(r io.Reader) *jsontext.Decoder { - return jsontext.NewDecoder(r, jsontext.AllowDuplicateNames(true), jsontext.AllowInvalidUTF8(true)) -} - -func expectDelim(dec *jsontext.Decoder, want jsontext.Kind) error { - tok, err := dec.ReadToken() - if err != nil { - return err - } - if tok.Kind() != want { - return fmt.Errorf("npm metadata: expected %v, got %v", want, tok.Kind()) - } - return nil -} - -// expectEnd checks nothing follows the top-level object. -func expectEnd(dec *jsontext.Decoder) error { - _, err := dec.ReadToken() - switch err { - case io.EOF: - return nil - case nil: - return errors.New("unexpected data after the top-level object") - } - return err -} - -func objectKey(dec *jsontext.Decoder) (string, error) { - tok, err := dec.ReadToken() - if err != nil { - return "", err - } - if tok.Kind() != '"' { - return "", fmt.Errorf("npm metadata: object key is %v, not a string", tok.Kind()) - } - return tok.String(), nil -} - -// decodeValue reads the next value into v, as encoding/json decodes it. -func decodeValue(dec *jsontext.Decoder, v any) error { - raw, err := dec.ReadValue() - if err != nil { - return err - } - return json.Unmarshal(raw, v) -} - -// objectKeys reads a value and returns its keys when it is an object. The -// members' values are skipped without being kept. -func objectKeys(dec *jsontext.Decoder) ([]string, bool, error) { - kind := dec.PeekKind() - if kind != '{' && kind != '[' { - return nil, false, dec.SkipValue() - } - if _, err := dec.ReadToken(); err != nil { - return nil, false, err - } - var keys []string - for next := dec.PeekKind(); next != '}' && next != ']'; next = dec.PeekKind() { - if kind == '{' { - key, err := objectKey(dec) - if err != nil { - return nil, false, err - } - keys = append(keys, key) - } - if err := dec.SkipValue(); err != nil { - return nil, false, err - } - } - if _, err := dec.ReadToken(); err != nil { - return nil, false, err - } - return keys, kind == '{', nil -} - -// VersionStream is Version for a packument too large to hold in memory. scan -// is the result of ScanPackument over the bytes r reads again; the second -// pass only looks for the one version entry it serves. For a document -// without a versions object it returns ErrNotStreamable. -func VersionStream(packageName, specifier string, scan *Scan, r io.Reader, options Options) ([]byte, error) { - if !scan.versionsObject || len(options.LocalVersions) > 0 { - return nil, ErrNotStreamable - } - h := &transformer{options} - f := scan.filter(h, packageName) - - target := specifier - if tag, ok := f.distTags[specifier].(string); ok { - target = tag - } - if options.DeniedVersions[target] { - return nil, ErrVersionDenied - } - if _, kept := f.versions[target]; !kept { - return nil, ErrVersionNotFound - } - entry, err := LookupVersion(r, target) - if err != nil { - return nil, err - } - if entry.Entry == nil { - return nil, ErrVersionNotFound - } - var out bytes.Buffer - h.writeNPMVersion(&out, packageName, target, entry.Entry) - return out.Bytes(), nil -} - -// VersionEntry is what a packument holds for one version. -type VersionEntry struct { - // Entry is versions[version], or nil when there is none. - Entry json.RawMessage - // Published is time[version], or "" when it is missing or not a string. - Published string - // Abbreviated is set for an abbreviated packument (see Abbreviated). - Abbreviated bool -} - -// LookupVersion reads a packument once and returns what it holds for -// version, keeping no other version's entry. Of repeated keys the last one -// counts, as with JSON.parse. -func LookupVersion(r io.Reader, version string) (VersionEntry, error) { - var e VersionEntry - var hasTime, hasModified bool - dec := newDecoder(r) - if err := expectDelim(dec, '{'); err != nil { - return e, err - } - for dec.PeekKind() != '}' { - key, err := objectKey(dec) - if err != nil { - return e, err - } - switch key { - case "versions": - e.Entry, err = objectMember(dec, version) - case "time": - hasTime = true - var raw json.RawMessage - raw, err = objectMember(dec, version) - e.Published = "" - if err == nil && raw != nil { - _ = json.Unmarshal(raw, &e.Published) - } - default: - hasModified = hasModified || key == "modified" - err = dec.SkipValue() - } - if err != nil { - return e, err - } - } - if err := expectDelim(dec, '}'); err != nil { - return e, err - } - if err := expectEnd(dec); err != nil { - return e, err - } - e.Abbreviated = hasModified && !hasTime - return e, nil -} - -// objectMember reads a value and returns its member key when it is an object -// that has one, the last if repeated, or nil. Other members are skipped -// without being kept. -func objectMember(dec *jsontext.Decoder, key string) (json.RawMessage, error) { - kind := dec.PeekKind() - if kind != '{' && kind != '[' { - return nil, dec.SkipValue() - } - if _, err := dec.ReadToken(); err != nil { - return nil, err - } - var found json.RawMessage - for next := dec.PeekKind(); next != '}' && next != ']'; next = dec.PeekKind() { - k := "" - if kind == '{' { - var err error - if k, err = objectKey(dec); err != nil { - return nil, err - } - } - if kind == '{' && k == key { - raw, err := dec.ReadValue() - if err != nil { - return nil, err - } - // ReadValue's bytes are only valid until the next read. - found = bytes.Clone(raw) - continue - } - if err := dec.SkipValue(); err != nil { - return nil, err - } - } - if _, err := dec.ReadToken(); err != nil { - return nil, err - } - return found, nil -} diff --git a/internal/metadata/npm/stream_bench_test.go b/internal/metadata/npm/stream_bench_test.go deleted file mode 100644 index 12d04d8..0000000 --- a/internal/metadata/npm/stream_bench_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package npm - -import ( - "bytes" - "io" - "os" - "testing" - - "github.com/git-pkgs/proxy/internal/metadata/npm/npmbench" -) - -func BenchmarkScanPackument(b *testing.B) { - body := npmbench.Packument(5000) - b.SetBytes(int64(len(body))) - b.ReportAllocs() - for b.Loop() { - if _, err := ScanPackument(bytes.NewReader(body)); err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkLookupVersion(b *testing.B) { - body := npmbench.Packument(5000) - b.SetBytes(int64(len(body))) - b.ReportAllocs() - for b.Loop() { - entry, err := LookupVersion(bytes.NewReader(body), "1.0.2500") - if err != nil || entry.Entry == nil || entry.Published == "" { - b.Fatalf("LookupVersion = %+v, %v", entry, err) - } - } -} - -// BenchmarkRewriteStream times RewriteStream's second pass, the copy, over -// the 5,000-version packument, or the one NPM_PACKUMENT_BENCH names. -func BenchmarkRewriteStream(b *testing.B) { - body := npmbench.Packument(5000) - name := "big" - if path := os.Getenv("NPM_PACKUMENT_BENCH"); path != "" { - var err error - if body, err = os.ReadFile(path); err != nil { - b.Fatal(err) - } - name = os.Getenv("NPM_PACKUMENT_BENCH_NAME") - } - scan, err := ScanPackument(bytes.NewReader(body)) - if err != nil { - b.Fatal(err) - } - options := Options{ProxyURL: "http://proxy.example"} - b.SetBytes(int64(len(body))) - b.ReportAllocs() - for b.Loop() { - if _, err := RewriteStream(name, scan, bytes.NewReader(body), io.Discard, options); err != nil { - b.Fatal(err) - } - } -} diff --git a/internal/metadata/npm/stream_test.go b/internal/metadata/npm/stream_test.go deleted file mode 100644 index 0bd93ae..0000000 --- a/internal/metadata/npm/stream_test.go +++ /dev/null @@ -1,254 +0,0 @@ -package npm - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "reflect" - "slices" - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -// rewriteStream runs both passes of RewriteStream over body. -func rewriteStream(packageName string, body []byte, options Options) ([]byte, []string, error) { - scan, err := ScanPackument(bytes.NewReader(body)) - if err != nil { - return nil, nil, err - } - var out bytes.Buffer - kept, err := RewriteStream(packageName, scan, bytes.NewReader(body), &out, options) - return out.Bytes(), kept, err -} - -// streamCases returns packuments with the filtered members in either -// order, duplicates and values Rewrite copies verbatim, and the options to -// rewrite them with. -func streamCases() (map[string]string, map[string]Options) { - now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) - ts := func(age time.Duration) string { return now.Add(-age).Format(time.RFC3339) } - version := func(v string) string { - return fmt.Sprintf(`%q:{"version":%q,"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-%s.tgz"}}`, v, v, v) - } - versions := `"versions":{` + version("1.0.0") + `,` + version("2.0.0") + `,` + version("3.0.0") + `}` - times := `"time":{"created":"` + ts(1000*time.Hour) + `","1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}` - tags := `"dist-tags":{"latest":"3.0.0","beta":"2.0.0","next":"1.0.0"}` - - docs := map[string]string{ - "testdata": string(packument), - "tags, versions, time": `{"name":"demo",` + tags + `,` + versions + `,` + times + `}`, - "time first": `{` + times + `,"name":"demo",` + versions + `,` + tags + `}`, - "no time": `{"name":"demo",` + tags + `,` + versions + `}`, - "time null": `{"time":null,` + tags + `,` + versions + `}`, - "tags not an object": `{"dist-tags":"x",` + versions + `,` + times + `}`, - "duplicates": `{"versions":{` + version("9.0.0") + `},` + tags + `,"time":{"9.0.0":"` + ts(time.Hour) + `"},` + - versions + `,` + times + `,"dist-tags":{"latest":"3.0.0"}}`, - "verbatim values": `{"_big":12345678901234567890,"readme":"&","nested":{"a":[1, 2 ,{"b":null}]},` + versions + `}`, - "unusual entries": `{"versions":{"1.0.0":"not an object","2.0.0":{"dist":"x"},"3.0.0":{"dist":{"tarball":7}},` + - `"4.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/renamed.tgz"}},"5.0.0":{"dist":{}}}}`, - "whitespace": "{ \"name\" : \"demo\" ,\n\t\"versions\" : {\n \"1.0.0\" : { \"dist\" : { \"tarball\" : \"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz\" } } } }", - "empty versions": `{"name":"demo","versions":{}}`, - } - options := map[string]Options{ - "plain": {ProxyURL: "http://proxy"}, - "cooldown": {ProxyURL: "http://proxy", Cooldown: &cooldown.Config{Default: "3d"}, EvaluatedAt: now}, - "denylist": {ProxyURL: "http://proxy", DeniedVersions: map[string]bool{"2.0.0": true}}, - "both": {ProxyURL: "http://proxy", Cooldown: &cooldown.Config{Default: "3d"}, EvaluatedAt: now, - DeniedVersions: map[string]bool{"1.0.0": true}}, - } - return docs, options -} - -// TestRewriteStreamMatchesRewrite checks that the streamed rewrite writes the -// same bytes and keeps the same versions as Rewrite, over documents with the -// filtered members in either order, duplicates, and values Rewrite copies -// verbatim. -func TestRewriteStreamMatchesRewrite(t *testing.T) { - docs, options := streamCases() - for docName, doc := range docs { - for optName, opts := range options { - t.Run(docName+"/"+optName, func(t *testing.T) { - want, wantKept, err := Rewrite("demo", []byte(doc), opts) - if err != nil { - t.Fatalf("Rewrite: %v", err) - } - got, gotKept, err := rewriteStream("demo", []byte(doc), opts) - if err != nil { - t.Fatalf("RewriteStream: %v", err) - } - // Rewrite keeps the whitespace inside the members of a - // filtered time map, which the stream doesn't see. - if !bytes.Equal(compact(t, got), compact(t, want)) { - t.Errorf("RewriteStream =\n%s\nRewrite =\n%s", got, want) - } - if docName == "verbatim values" && !bytes.Contains(got, []byte(`"_big":12345678901234567890,"readme":"&","nested":{"a":[1, 2 ,{"b":null}]}`)) { - t.Errorf("values not copied verbatim: %s", got) - } - slices.Sort(gotKept) - slices.Sort(wantKept) - if !slices.Equal(gotKept, wantKept) { - t.Errorf("kept %v, Rewrite kept %v", gotKept, wantKept) - } - }) - } - } -} - -func compact(t *testing.T, doc []byte) []byte { - t.Helper() - var out bytes.Buffer - if err := json.Compact(&out, doc); err != nil { - t.Fatalf("output is not JSON: %v\n%s", err, doc) - } - return out.Bytes() -} - -func TestRewriteStreamCallbacks(t *testing.T) { - var filtered []string - var rewritten []int - options := Options{ProxyURL: "http://proxy", Cooldown: &cooldown.Config{Default: "7d"}, - EvaluatedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), - OnCooldownFilter: func(version, _ string) { filtered = append(filtered, version) }, - OnTarballsRewritten: func(count int) { rewritten = append(rewritten, count) }, - } - if _, _, err := rewriteStream("@example/widgets", packument, options); err != nil { - t.Fatal(err) - } - if !slices.Equal(filtered, []string{"3.0.0"}) || !slices.Equal(rewritten, []int{2}) { - t.Errorf("filtered %v, rewritten %v; want [3.0.0] and [2]", filtered, rewritten) - } -} - -func TestRewriteStreamNotStreamable(t *testing.T) { - for name, tc := range map[string]struct { - doc string - options Options - }{ - "single version": {`{"name":"demo","version":"1.0.0","dist":{"tarball":"t"}}`, Options{}}, - "versions array": {`{"name":"demo","versions":[{"a":1}]}`, Options{}}, - "versions null": {`{"name":"demo","versions":null}`, Options{}}, - "pushed versions": {`{"versions":{}}`, Options{LocalVersions: []LocalVersion{{Version: "1.0.0", Entry: []byte(`{}`)}}}}, - } { - t.Run(name, func(t *testing.T) { - var out bytes.Buffer - scan, err := ScanPackument(strings.NewReader(tc.doc)) - if err != nil { - t.Fatal(err) - } - if _, err := RewriteStream("demo", scan, strings.NewReader(tc.doc), &out, tc.options); !errors.Is(err, ErrNotStreamable) { - t.Errorf("err = %v, want ErrNotStreamable", err) - } - if out.Len() != 0 { - t.Errorf("wrote %q before refusing", out.String()) - } - }) - } -} - -func TestScanPackument(t *testing.T) { - for _, body := range []string{`{"versions": {}`, `[1]`, `{"versions": {}} trailing`, `{"a":}`, ``} { - if _, err := ScanPackument(strings.NewReader(body)); err == nil { - t.Errorf("ScanPackument(%q): expected an error", body) - } - } - for body, want := range map[string]bool{ - `{"name":"demo","modified":"2024-01-01T00:00:00Z","versions":{}}`: true, - `{"name":"demo","modified":"2024-01-01T00:00:00Z","time":{},"versions":{}}`: false, - `{"name":"demo","versions":{}}`: false, - } { - scan, err := ScanPackument(strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } - if scan.Abbreviated() != want || Abbreviated([]byte(body)) != want { - t.Errorf("Abbreviated(%s) = %v, want %v", body, scan.Abbreviated(), want) - } - } -} - -// TestVersionStreamMatchesVersion checks that a single version resolved from -// the stream is the one Version serves, with the same errors. -func TestVersionStreamMatchesVersion(t *testing.T) { - docs, options := streamCases() - specifiers := []string{"latest", "beta", "next", "old", "1.0.0", "2.0.0", "3.0.0", "9.0.0", "missing"} - for docName, doc := range docs { - scan, err := ScanPackument(strings.NewReader(doc)) - if err != nil { - t.Fatalf("%s: %v", docName, err) - } - for optName, opts := range options { - for _, spec := range specifiers { - want, wantErr := Version("demo", spec, []byte(doc), opts) - got, gotErr := VersionStream("demo", spec, scan, strings.NewReader(doc), opts) - name := docName + "/" + optName + "/" + spec - switch { - case wantErr != nil || gotErr != nil: - if !errors.Is(gotErr, ErrVersionDenied) == errors.Is(wantErr, ErrVersionDenied) || - !errors.Is(gotErr, ErrVersionNotFound) == errors.Is(wantErr, ErrVersionNotFound) || - (gotErr == nil) != (wantErr == nil) { - t.Errorf("%s: error %v, Version gave %v", name, gotErr, wantErr) - } - case !bytes.Equal(got, want): - t.Errorf("%s:\n%s\nVersion:\n%s", name, got, want) - } - } - } - } -} - -func TestLookupVersion(t *testing.T) { - doc := `{"modified":"x","versions":{"1.0.0":{"a":1}},"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":7},` + - `"versions":{"1.0.0":{"a":2},"2.0.0":{"b":1}}}` - for version, want := range map[string]VersionEntry{ - "1.0.0": {Entry: json.RawMessage(`{"a":2}`), Published: "2020-01-01T00:00:00Z"}, - "2.0.0": {Entry: json.RawMessage(`{"b":1}`)}, - "3.0.0": {}, - } { - got, err := LookupVersion(strings.NewReader(doc), version) - if err != nil { - t.Fatal(err) - } - if string(got.Entry) != string(want.Entry) || got.Published != want.Published || got.Abbreviated { - t.Errorf("LookupVersion(%s) = %+v, want %+v", version, got, want) - } - } - got, err := LookupVersion(strings.NewReader(`{"modified":"x","versions":{}}`), "1.0.0") - if err != nil || !got.Abbreviated { - t.Errorf("abbreviated packument: %+v, %v", got, err) - } - if _, err := LookupVersion(strings.NewReader(`{"versions":{}`), "1.0.0"); err == nil { - t.Error("expected an error for truncated JSON") - } -} - -// TestStreamReadsJSONAsEncodingJSON checks what the streaming readers accept -// matches what Rewrite's encoding/json does: names repeated at any depth, -// invalid UTF-8 in strings, and a version entry's bytes kept as they are. -func TestStreamReadsJSONAsEncodingJSON(t *testing.T) { - entry := "{ \"name\" : \"demo\",\n \"x\":1, \"x\":2, \"note\":\"caf\xe9\" }" - doc := "{\"readme\":\"bad \xff byte\",\"meta\":{\"k\":1,\"k\":2},\"dist-tags\":{\"latest\":\"1.0.0\",\"latest\":\"1.0.0\"}," + - "\"versions\":{\"1.0.0\":" + entry + "},\"time\":{\"1.0.0\":\"2020-01-01T00:00:00Z\"}}\n" - if !json.Valid([]byte(strings.ToValidUTF8(doc, "?"))) { - t.Fatal("test document isn't JSON") - } - - scan, err := ScanPackument(strings.NewReader(doc)) - if err != nil { - t.Fatalf("ScanPackument: %v", err) - } - if !slices.Equal(scan.versions, []string{"1.0.0"}) || !reflect.DeepEqual(scan.distTags, map[string]any{"latest": "1.0.0"}) { - t.Errorf("scan versions %v, dist-tags %v", scan.versions, scan.distTags) - } - - got, err := LookupVersion(strings.NewReader(doc), "1.0.0") - if err != nil { - t.Fatalf("LookupVersion: %v", err) - } - if string(got.Entry) != entry || got.Published != "2020-01-01T00:00:00Z" { - t.Errorf("entry %q, published %q; want the entry's bytes %q", got.Entry, got.Published, entry) - } -} diff --git a/internal/metadata/npm/testdata/packument.json b/internal/metadata/npm/testdata/packument.json deleted file mode 100644 index cdb36b5..0000000 --- a/internal/metadata/npm/testdata/packument.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "@example/widgets", - "dist-tags": {"latest": "3.0.0", "beta": "2.0.0"}, - "versions": { - "1.0.0": {"name": "@example/widgets", "version": "1.0.0", "dependencies": {"left-pad": "^1.3.0"}, "dist": {"integrity": "sha512-example", "tarball": "https://registry.npmjs.org/@example/widgets/-/widgets-1.0.0.tgz"}}, - "2.0.0": {"name": "@example/widgets", "version": "2.0.0", "dist": {"tarball": "https://registry.npmjs.org/@example/widgets/-/widgets-2.0.0.tgz"}}, - "3.0.0": {"name": "@example/widgets", "version": "3.0.0", "dist": {"tarball": "https://cdn.example/abcdef.tgz"}} - }, - "time": {"created": "2020-01-01T00:00:00Z", "1.0.0": "2020-01-01T00:00:00Z", "2.0.0": "2021-01-01T00:00:00Z", "3.0.0": "2099-01-01T00:00:00Z"}, - "readme": "Widgets & tools", - "downloads": 12345678901234567890 -} diff --git a/internal/metrics/analytics_test.go b/internal/metrics/analytics_test.go deleted file mode 100644 index 427c38f..0000000 --- a/internal/metrics/analytics_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package metrics - -import ( - "testing" - - "github.com/prometheus/client_golang/prometheus/testutil" -) - -func TestUpdateEcosystemStats(t *testing.T) { - UpdateEcosystemStats([]EcosystemStats{ - {Ecosystem: "npm", Packages: 10, Versions: 25, Artifacts: 40, CacheSize: 1500, Downloads: 4, DownloadedBytes: 3500}, - {Ecosystem: "cargo", Packages: 2, Versions: 3, Artifacts: 3, CacheSize: 200, Downloads: 10, DownloadedBytes: 2000}, - }) - - if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("npm")); got != 3500 { - t.Errorf("npm downloaded bytes = %v, want 3500", got) - } - if got := testutil.ToFloat64(EcosystemDownloads.WithLabelValues("npm")); got != 4 { - t.Errorf("npm downloads = %v, want 4", got) - } - if got := testutil.ToFloat64(EcosystemCacheSize.WithLabelValues("cargo")); got != 200 { - t.Errorf("cargo cache size = %v, want 200", got) - } - if got := testutil.ToFloat64(EcosystemCachedArtifacts.WithLabelValues("npm")); got != 40 { - t.Errorf("npm cached artifacts = %v, want 40", got) - } - if got := testutil.ToFloat64(EcosystemPackages.WithLabelValues("npm")); got != 10 { - t.Errorf("npm packages = %v, want 10", got) - } - if got := testutil.ToFloat64(EcosystemVersions.WithLabelValues("npm")); got != 25 { - t.Errorf("npm versions = %v, want 25", got) - } -} - -// A refresh that no longer mentions an ecosystem must drop its series rather -// than leave it frozen at the last observed value. -func TestUpdateEcosystemStatsDropsStaleSeries(t *testing.T) { - UpdateEcosystemStats([]EcosystemStats{ - {Ecosystem: "npm", DownloadedBytes: 3500}, - {Ecosystem: "gem", DownloadedBytes: 900}, - }) - if got := testutil.CollectAndCount(EcosystemDownloadedBytes); got != 2 { - t.Fatalf("expected 2 series after first refresh, got %d", got) - } - - UpdateEcosystemStats([]EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 4000}}) - - if got := testutil.CollectAndCount(EcosystemDownloadedBytes); got != 1 { - t.Errorf("expected 1 series after gem disappeared, got %d", got) - } - if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("npm")); got != 4000 { - t.Errorf("npm downloaded bytes = %v, want 4000", got) - } -} - -// Ecosystem labels are normalized so these gauges join against the other -// metrics that take their ecosystem from a package record. -func TestUpdateEcosystemStatsNormalizesLabels(t *testing.T) { - UpdateEcosystemStats([]EcosystemStats{{Ecosystem: "NPM", DownloadedBytes: 12}}) - - if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("npm")); got != 12 { - t.Errorf("normalized npm gauge = %v, want 12", got) - } -} - -// GetEcosystemStats groups by the raw packages.ecosystem column, so a database -// carrying both spellings of an aliased ecosystem yields two rows that -// normalize to one label. They must sum rather than overwrite each other. -func TestUpdateEcosystemStatsSumsAliasedRows(t *testing.T) { - UpdateEcosystemStats([]EcosystemStats{ - {Ecosystem: "gem", DownloadedBytes: 100, CacheSize: 10, Packages: 1}, - {Ecosystem: "rubygems", DownloadedBytes: 200, CacheSize: 20, Packages: 2}, - {Ecosystem: "npm", DownloadedBytes: 50, CacheSize: 5, Packages: 3}, - }) - - if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("rubygems")); got != 300 { - t.Errorf("rubygems downloaded bytes = %v, want 300", got) - } - if got := testutil.ToFloat64(EcosystemCacheSize.WithLabelValues("rubygems")); got != 30 { - t.Errorf("rubygems cache size = %v, want 30", got) - } - if got := testutil.ToFloat64(EcosystemPackages.WithLabelValues("rubygems")); got != 3 { - t.Errorf("rubygems packages = %v, want 3", got) - } - // An unaliased ecosystem is unaffected. - if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("npm")); got != 50 { - t.Errorf("npm downloaded bytes = %v, want 50", got) - } - - // A later refresh replaces the value rather than adding to it. - UpdateEcosystemStats([]EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 50}}) - if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("npm")); got != 50 { - t.Errorf("npm downloaded bytes after a second refresh = %v, want 50", got) - } -} diff --git a/internal/metrics/failures.go b/internal/metrics/failures.go deleted file mode 100644 index a5f276b..0000000 --- a/internal/metrics/failures.go +++ /dev/null @@ -1,113 +0,0 @@ -package metrics - -import ( - "net/url" - "regexp" - "sync" - "time" -) - -// recentFailuresPerSeries bounds how many failures are kept for each series of -// a failure counter. The series set is already bounded by the counter's -// labels, so this caps the whole store. -const recentFailuresPerSeries = 20 - -// FailureDetail is what a failure counter's call site knows beyond the -// counter's labels: which artifact failed, where it came from or was going, -// and why. Every field is optional. -// -// None of it becomes a metric label. Package names and URLs are unbounded, so -// they are kept in a small in-memory store for /ui/analytics instead. -type FailureDetail struct { - PURL string - Filename string - URL string - Path string - Err error -} - -// Failure is one recorded failure, with credentials already stripped from the -// URL and the error text. -type Failure struct { - At time.Time - PURL string - Filename string - URL string - Path string - Error string -} - -// recentFailures holds the latest failures per counter series. It lives in -// process memory only, so like the counters themselves it starts empty when -// the proxy starts. -var recentFailures = struct { - sync.Mutex - series map[string][]Failure -}{series: make(map[string][]Failure)} - -// RecentFailures returns the latest failures recorded for one series of a -// failure counter, newest first, or nil when the series has none. -func RecentFailures(name string, labels map[string]string) []Failure { - recentFailures.Lock() - defer recentFailures.Unlock() - - stored := recentFailures.series[failureKey(name, labels)] - if len(stored) == 0 { - return nil - } - return append([]Failure(nil), stored...) -} - -func recordFailure(name string, labels map[string]string, detail FailureDetail) { - failure := Failure{ - At: time.Now(), - PURL: detail.PURL, - Filename: detail.Filename, - URL: redactURL(detail.URL), - Path: detail.Path, - } - if detail.Err != nil { - failure.Error = redactURLsIn(detail.Err.Error()) - } - - key := failureKey(name, labels) - - recentFailures.Lock() - defer recentFailures.Unlock() - - stored := append([]Failure{failure}, recentFailures.series[key]...) - if len(stored) > recentFailuresPerSeries { - stored = stored[:recentFailuresPerSeries] - } - recentFailures.series[key] = stored -} - -func failureKey(name string, labels map[string]string) string { - return name + "{" + labelKey(labels) + "}" -} - -var urlInText = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9+.-]*://[^\s"'<>]+`) - -// redactURLsIn strips credentials from every URL embedded in text, such as the -// one net/http quotes in a failed request's error. -func redactURLsIn(text string) string { - return urlInText.ReplaceAllStringFunc(text, redactURL) -} - -// redactURL drops the userinfo, query and fragment, which is where registry -// tokens and signed-URL signatures live. -func redactURL(raw string) string { - if raw == "" { - return "" - } - u, err := url.Parse(raw) - if err != nil { - return "(unparseable URL)" - } - u.User = nil - u.RawQuery = "" - u.ForceQuery = false - u.Fragment = "" - u.RawFragment = "" - return u.String() -} diff --git a/internal/metrics/failures_test.go b/internal/metrics/failures_test.go deleted file mode 100644 index 2301f8b..0000000 --- a/internal/metrics/failures_test.go +++ /dev/null @@ -1,139 +0,0 @@ -package metrics - -import ( - "errors" - "fmt" - "strings" - "testing" - - "github.com/prometheus/client_golang/prometheus/testutil" -) - -// isolateRecentFailures empties the process-wide failure history for the -// test and again when it ends, so neither earlier tests nor repeated runs -// reach its assertions. -func isolateRecentFailures(t *testing.T) { - t.Helper() - reset := func() { - recentFailures.Lock() - defer recentFailures.Unlock() - recentFailures.series = make(map[string][]Failure) - } - reset() - t.Cleanup(reset) -} - -func TestRecentFailuresNewestFirst(t *testing.T) { - isolateRecentFailures(t) - - RecordUpstreamError("npm", "fetch_failed", FailureDetail{PURL: "pkg:npm/left-pad@1.0.0"}) - RecordUpstreamError("npm", "fetch_failed", FailureDetail{PURL: "pkg:npm/right-pad@2.0.0"}) - - got := RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "npm", "error_type": "fetch_failed"}) - if len(got) != 2 { - t.Fatalf("got %d failures, want 2", len(got)) - } - if got[0].PURL != "pkg:npm/right-pad@2.0.0" || got[1].PURL != "pkg:npm/left-pad@1.0.0" { - t.Errorf("failures not newest first: %q, %q", got[0].PURL, got[1].PURL) - } - if got[0].At.IsZero() { - t.Error("failure has no timestamp") - } -} - -func TestRecentFailuresAreBoundedPerSeries(t *testing.T) { - isolateRecentFailures(t) - - for i := range recentFailuresPerSeries + 5 { - RecordStorageError("write", FailureDetail{Path: fmt.Sprintf("npm/pkg-%d.tgz", i)}) - } - - got := RecentFailures("proxy_storage_errors_total", map[string]string{"operation": "write"}) - if len(got) != recentFailuresPerSeries { - t.Fatalf("got %d failures, want the cap of %d", len(got), recentFailuresPerSeries) - } - if want := fmt.Sprintf("npm/pkg-%d.tgz", recentFailuresPerSeries+4); got[0].Path != want { - t.Errorf("newest failure = %q, want %q", got[0].Path, want) - } -} - -func TestRecentFailuresAreKeptPerSeries(t *testing.T) { - isolateRecentFailures(t) - - RecordIntegrityFailure("npm", FailureDetail{PURL: "pkg:npm/a@1"}) - RecordIntegrityFailure("pypi", FailureDetail{PURL: "pkg:pypi/b@1"}) - - npm := RecentFailures("proxy_integrity_failures_total", map[string]string{"ecosystem": "npm"}) - if len(npm) != 1 || npm[0].PURL != "pkg:npm/a@1" { - t.Errorf("npm series = %+v, want only pkg:npm/a@1", npm) - } - if got := RecentFailures("proxy_integrity_failures_total", map[string]string{"ecosystem": "deb"}); got != nil { - t.Errorf("unseen series = %+v, want nil", got) - } -} - -func TestRecordFailureStillCountsTheMetric(t *testing.T) { - isolateRecentFailures(t) - - before := testutil.ToFloat64(HealthProbeFailures.WithLabelValues("read")) - RecordHealthProbeFailure("read", FailureDetail{Err: errors.New("disk gone")}) - if diff := testutil.ToFloat64(HealthProbeFailures.WithLabelValues("read")) - before; diff != 1 { - t.Errorf("counter delta = %.0f, want 1", diff) - } - - got := RecentFailures("proxy_health_probe_failures_total", map[string]string{"step": "read"}) - if len(got) != 1 || got[0].Error != "disk gone" { - t.Errorf("failures = %+v, want one carrying the error text", got) - } -} - -func TestScanErrorRecordsDetail(t *testing.T) { - isolateRecentFailures(t) - - RecordScanError("NPM", "clamav", "timeout", FailureDetail{PURL: "pkg:npm/a@1", Filename: "a-1.tgz"}) - - got := RecentFailures("proxy_scan_errors_total", - map[string]string{"ecosystem": "npm", "scanner": "clamav", "error_type": "timeout"}) - if len(got) != 1 || got[0].Filename != "a-1.tgz" { - t.Errorf("failures = %+v, want one keyed by the normalised ecosystem", got) - } -} - -// The analytics page is unauthenticated, and private registry URLs carry -// credentials in the userinfo or the query string. Neither may be kept. -func TestRecentFailuresStripCredentials(t *testing.T) { - isolateRecentFailures(t) - - RecordUpstreamError("npm", "fetch_failed", FailureDetail{ - URL: "https://user:s3cret@registry.example.com/pkg/-/pkg-1.0.0.tgz?token=abc#frag", - Err: errors.New(`Get "https://user:s3cret@registry.example.com/pkg?sig=xyz": connection refused`), - }) - - got := RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "npm", "error_type": "fetch_failed"}) - if len(got) != 1 { - t.Fatalf("got %d failures, want 1", len(got)) - } - if want := "https://registry.example.com/pkg/-/pkg-1.0.0.tgz"; got[0].URL != want { - t.Errorf("URL = %q, want %q", got[0].URL, want) - } - if want := `Get "https://registry.example.com/pkg": connection refused`; got[0].Error != want { - t.Errorf("Error = %q, want %q", got[0].Error, want) - } - for _, secret := range []string{"s3cret", "token", "sig=", "user:"} { - if strings.Contains(got[0].URL+got[0].Error, secret) { - t.Errorf("recorded failure still contains %q", secret) - } - } -} - -func TestRecentFailuresReturnsACopy(t *testing.T) { - isolateRecentFailures(t) - - RecordStorageError("read", FailureDetail{Path: "a"}) - labels := map[string]string{"operation": "read"} - RecentFailures("proxy_storage_errors_total", labels)[0].Path = "mutated" - - if got := RecentFailures("proxy_storage_errors_total", labels)[0].Path; got != "a" { - t.Errorf("stored failure was mutated through the returned slice: %q", got) - } -} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 85eeef4..f23a5a9 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -4,10 +4,8 @@ package metrics import ( "net/http" "strconv" - "sync" "time" - "github.com/git-pkgs/purl" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -35,23 +33,15 @@ var ( CacheHits = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "proxy_cache_hits_total", - Help: "Total number of cache hits by ecosystem and kind (artifact or metadata)", + Help: "Total number of cache hits by ecosystem", }, - []string{"ecosystem", "kind"}, + []string{"ecosystem"}, ) CacheMisses = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "proxy_cache_misses_total", - Help: "Total number of cache misses by ecosystem and kind (artifact or metadata)", - }, - []string{"ecosystem", "kind"}, - ) - - MetadataServedBytes = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_metadata_cache_served_bytes_total", - Help: "Total metadata bytes served from the metadata cache, by ecosystem", + Help: "Total number of cache misses by ecosystem", }, []string{"ecosystem"}, ) @@ -70,38 +60,6 @@ var ( }, ) - ArtifactsEvicted = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_artifacts_evicted_total", - Help: "Total number of cached artifacts evicted, by reason (lru or retention) and ecosystem", - }, - []string{"reason", "ecosystem"}, - ) - - MetadataEvicted = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_metadata_evicted_total", - Help: "Total number of cached metadata documents evicted by the size limit, by ecosystem", - }, - []string{"ecosystem"}, - ) - - MetadataExpiredByChangeFeed = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_metadata_expired_by_change_feed_total", - Help: "Cached metadata documents set to revalidate because an upstream change feed listed them, by ecosystem", - }, - []string{"ecosystem"}, - ) - - FetchLeaseWaits = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_fetch_lease_waits_total", - Help: "Cache misses that waited for another instance's upstream fetch, by kind (artifact or metadata)", - }, - []string{"kind"}, - ) - // Upstream metrics UpstreamFetchDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -137,14 +95,6 @@ var ( []string{"registry"}, ) - UpstreamRateLimitHolds = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_upstream_rate_limit_holds_total", - Help: "Holds placed on an upstream host after it rate limited the proxy", - }, - []string{"host"}, - ) - // Storage metrics StorageOperationDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -171,14 +121,6 @@ var ( }, ) - ArtifactsServedUncached = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_artifacts_served_uncached_total", - Help: "Artifacts sent to the client on a cache miss that could not be cached, such as when storage is full", - }, - []string{"ecosystem"}, - ) - IntegrityFailures = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "proxy_integrity_failures_total", @@ -194,125 +136,6 @@ var ( }, []string{"step"}, ) - - // Per-ecosystem gauges, derived from the database rather than incremented - // in the request path, and refreshed on the same tick as the cache gauges - // above. - EcosystemDownloadedBytes = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "proxy_ecosystem_downloaded_bytes", - Help: "Accumulated bytes served from cache per ecosystem (cache hits x artifact size)", - }, - []string{"ecosystem"}, - ) - - EcosystemDownloads = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "proxy_ecosystem_artifact_downloads", - Help: "Accumulated artifact downloads served from cache per ecosystem", - }, - []string{"ecosystem"}, - ) - - EcosystemCacheSize = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "proxy_ecosystem_cache_size_bytes", - Help: "Size of cached artifacts per ecosystem in bytes", - }, - []string{"ecosystem"}, - ) - - EcosystemCachedArtifacts = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "proxy_ecosystem_cached_artifacts", - Help: "Number of cached artifacts per ecosystem", - }, - []string{"ecosystem"}, - ) - - EcosystemPackages = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "proxy_ecosystem_packages", - Help: "Number of known packages per ecosystem", - }, - []string{"ecosystem"}, - ) - - EcosystemVersions = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "proxy_ecosystem_versions", - Help: "Number of known package versions per ecosystem", - }, - []string{"ecosystem"}, - ) - - // Metadata cache gauges, derived from the metadata_cache table on the - // same tick as the per-ecosystem gauges above. - MetadataCacheSize = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "proxy_metadata_cache_size_bytes", - Help: "Size of cached metadata documents per ecosystem in bytes", - }, - []string{"ecosystem"}, - ) - - MetadataCacheEntries = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "proxy_metadata_cache_entries", - Help: "Number of cached metadata documents per ecosystem", - }, - []string{"ecosystem"}, - ) - - ResponseBytes = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_response_bytes_total", - Help: "Total response body bytes written to clients, by ecosystem", - }, - []string{"ecosystem"}, - ) - - ClientRequests = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_client_requests_total", - Help: "Total requests by client tool, as identified from the User-Agent", - }, - []string{"client"}, - ) - - ClientResponseBytes = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_client_response_bytes_total", - Help: "Total response body bytes written to clients, by client tool", - }, - []string{"client"}, - ) - - // Scanning metrics - ScanDuration = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "proxy_scan_duration_seconds", - Help: "Pre-cache artifact scan duration in seconds, by ecosystem and scanner", - Buckets: prometheus.DefBuckets, - }, - []string{"ecosystem", "scanner"}, - ) - - ScanBlocked = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_scan_blocked_total", - Help: "Total number of artifacts blocked by a pre-cache scan, by ecosystem and scanner", - }, - []string{"ecosystem", "scanner"}, - ) - - ScanErrors = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "proxy_scan_errors_total", - Help: "Total number of pre-cache scan errors, by ecosystem, scanner, and error type", - }, - []string{"ecosystem", "scanner", "error_type"}, - ) ) func init() { @@ -322,38 +145,17 @@ func init() { RequestDuration, CacheHits, CacheMisses, - MetadataServedBytes, CacheSize, CachedArtifacts, - ArtifactsEvicted, - MetadataEvicted, - MetadataExpiredByChangeFeed, - FetchLeaseWaits, UpstreamFetchDuration, UpstreamErrors, CircuitBreakerState, CircuitBreakerTrips, - UpstreamRateLimitHolds, StorageOperationDuration, StorageErrors, - ArtifactsServedUncached, ActiveRequests, IntegrityFailures, HealthProbeFailures, - EcosystemDownloadedBytes, - EcosystemDownloads, - EcosystemCacheSize, - EcosystemCachedArtifacts, - EcosystemPackages, - EcosystemVersions, - MetadataCacheSize, - MetadataCacheEntries, - ResponseBytes, - ClientRequests, - ClientResponseBytes, - ScanDuration, - ScanBlocked, - ScanErrors, ) } @@ -369,85 +171,14 @@ func RecordRequest(ecosystem string, status int, duration time.Duration) { RequestDuration.WithLabelValues(ecosystem, statusStr).Observe(duration.Seconds()) } -// RecordResponse tracks what a client actually downloaded. -// -// Distinct from proxy_ecosystem_downloaded_bytes: that gauge is derived from -// the database and counts cache hits multiplied by artifact size, while this -// counts body bytes as they are written, including metadata responses and -// cache misses. -// -// client must come from a closed set -- a User-Agent is attacker-controlled, so -// passing it through raw would mint a time series per request. -func RecordResponse(ecosystem, client string, bytes int64) { - ClientRequests.WithLabelValues(client).Inc() - if bytes <= 0 { - return - } - ResponseBytes.WithLabelValues(ecosystem).Add(float64(bytes)) - ClientResponseBytes.WithLabelValues(client).Add(float64(bytes)) +// RecordCacheHit increments cache hit counter. +func RecordCacheHit(ecosystem string) { + CacheHits.WithLabelValues(ecosystem).Inc() } -// Values of the kind label on proxy_cache_hits_total and -// proxy_cache_misses_total. Artifacts and metadata are separate caches with -// very different costs, so a hit on one says nothing about the other. -const ( - CacheKindArtifact = "artifact" - CacheKindMetadata = "metadata" -) - -// RecordArtifactCacheHit counts an artifact served from storage. -func RecordArtifactCacheHit(ecosystem string) { - CacheHits.WithLabelValues(purl.NormalizeEcosystem(ecosystem), CacheKindArtifact).Inc() -} - -// RecordMetadataCacheHit counts a metadata document served from the metadata -// cache, along with the bytes it served. -func RecordMetadataCacheHit(ecosystem string, bytes int) { - ecosystem = purl.NormalizeEcosystem(ecosystem) - CacheHits.WithLabelValues(ecosystem, CacheKindMetadata).Inc() - if bytes > 0 { - MetadataServedBytes.WithLabelValues(ecosystem).Add(float64(bytes)) - } -} - -// RecordArtifactEvicted counts one evicted artifact. ecosystem is the -// package record's value, normalized like the cache hit counter; an artifact -// without a package record counts as "other". -func RecordArtifactEvicted(reason, ecosystem string) { - if ecosystem == "" { - ecosystem = "other" - } - ArtifactsEvicted.WithLabelValues(reason, purl.NormalizeEcosystem(ecosystem)).Inc() -} - -// RecordMetadataEvicted counts one metadata document evicted by the size -// limit. ecosystem is the metadata cache's, normalized like the cache hit -// counter. -func RecordMetadataEvicted(ecosystem string) { - MetadataEvicted.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() -} - -// RecordMetadataExpiredByChangeFeed counts n metadata documents an upstream -// change feed listed as changed. ecosystem is the metadata cache's, -// normalized like the cache hit counter. -func RecordMetadataExpiredByChangeFeed(ecosystem string, n int) { - MetadataExpiredByChangeFeed.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Add(float64(n)) -} - -// RecordFetchLeaseWait counts a cache miss that found another instance -// sharing the database already fetching it. -func RecordFetchLeaseWait(kind string) { - FetchLeaseWaits.WithLabelValues(kind).Inc() -} - -// RecordArtifactCacheMiss counts an artifact that had to be fetched upstream. -func RecordArtifactCacheMiss(ecosystem string) { - CacheMisses.WithLabelValues(purl.NormalizeEcosystem(ecosystem), CacheKindArtifact).Inc() -} - -// RecordMetadataCacheMiss counts a metadata lookup the cache could not serve. -func RecordMetadataCacheMiss(ecosystem string) { - CacheMisses.WithLabelValues(purl.NormalizeEcosystem(ecosystem), CacheKindMetadata).Inc() +// RecordCacheMiss increments cache miss counter. +func RecordCacheMiss(ecosystem string) { + CacheMisses.WithLabelValues(ecosystem).Inc() } // RecordUpstreamFetch tracks upstream fetch duration. @@ -455,12 +186,9 @@ func RecordUpstreamFetch(ecosystem string, duration time.Duration) { UpstreamFetchDuration.WithLabelValues(ecosystem).Observe(duration.Seconds()) } -// RecordUpstreamError increments upstream error counter and keeps the detail -// for /ui/analytics. -func RecordUpstreamError(ecosystem, errorType string, detail FailureDetail) { +// RecordUpstreamError increments upstream error counter. +func RecordUpstreamError(ecosystem, errorType string) { UpstreamErrors.WithLabelValues(ecosystem, errorType).Inc() - recordFailure("proxy_upstream_errors_total", - map[string]string{"ecosystem": ecosystem, "error_type": errorType}, detail) } // RecordStorageOperation tracks storage operation duration. @@ -468,53 +196,20 @@ func RecordStorageOperation(operation string, duration time.Duration) { StorageOperationDuration.WithLabelValues(operation).Observe(duration.Seconds()) } -// RecordIntegrityFailure increments the integrity failure counter and keeps -// the detail for /ui/analytics. -func RecordIntegrityFailure(ecosystem string, detail FailureDetail) { +// RecordIntegrityFailure increments the integrity failure counter. +func RecordIntegrityFailure(ecosystem string) { IntegrityFailures.WithLabelValues(ecosystem).Inc() - recordFailure("proxy_integrity_failures_total", map[string]string{"ecosystem": ecosystem}, detail) } -// RecordHealthProbeFailure increments the health probe failure counter and -// keeps the detail for /ui/analytics. +// RecordHealthProbeFailure increments the health probe failure counter. // step is one of: "write", "size", "read", "verify", "delete". -func RecordHealthProbeFailure(step string, detail FailureDetail) { +func RecordHealthProbeFailure(step string) { HealthProbeFailures.WithLabelValues(step).Inc() - recordFailure("proxy_health_probe_failures_total", map[string]string{"step": step}, detail) } -// RecordStorageError increments storage error counter and keeps the detail -// for /ui/analytics. -func RecordStorageError(operation string, detail FailureDetail) { +// RecordStorageError increments storage error counter. +func RecordStorageError(operation string) { StorageErrors.WithLabelValues(operation).Inc() - recordFailure("proxy_storage_errors_total", map[string]string{"operation": operation}, detail) -} - -// RecordArtifactServedUncached counts an artifact a client got on a cache -// miss although storing it failed, and keeps the detail for /ui/analytics. -func RecordArtifactServedUncached(ecosystem string, detail FailureDetail) { - ecosystem = purl.NormalizeEcosystem(ecosystem) - ArtifactsServedUncached.WithLabelValues(ecosystem).Inc() - recordFailure("proxy_artifacts_served_uncached_total", map[string]string{"ecosystem": ecosystem}, detail) -} - -// RecordScanResult tracks a completed pre-cache scan call. -func RecordScanResult(ecosystem, scannerName string, allowed bool, duration time.Duration) { - ecosystem = purl.NormalizeEcosystem(ecosystem) - ScanDuration.WithLabelValues(ecosystem, scannerName).Observe(duration.Seconds()) - if !allowed { - ScanBlocked.WithLabelValues(ecosystem, scannerName).Inc() - } -} - -// RecordScanError increments the scan error counter. -// errorType is one of: "error" (scanner call failed), "timeout", "cancelled". -// The detail is kept for /ui/analytics. -func RecordScanError(ecosystem, scannerName, errorType string, detail FailureDetail) { - ecosystem = purl.NormalizeEcosystem(ecosystem) - ScanErrors.WithLabelValues(ecosystem, scannerName, errorType).Inc() - recordFailure("proxy_scan_errors_total", - map[string]string{"ecosystem": ecosystem, "scanner": scannerName, "error_type": errorType}, detail) } // UpdateCacheStats updates cache size and artifact count gauges. @@ -523,123 +218,6 @@ func UpdateCacheStats(sizeBytes, artifactCount int64) { CachedArtifacts.Set(float64(artifactCount)) } -// EcosystemStats is one ecosystem's row of the snapshot published as gauges. -type EcosystemStats struct { - Ecosystem string - Packages int64 - Versions int64 - Artifacts int64 - CacheSize int64 - Downloads int64 - DownloadedBytes int64 -} - -// publishedEcosystems tracks which labels the per-ecosystem gauges currently -// carry, so a label that disappears can be deleted individually. -var ( - publishedMu sync.Mutex - publishedEcosystems = map[string]bool{} - // publishedMetadataEcosystems does the same for the metadata cache gauges. - publishedMetadataEcosystems = map[string]bool{} -) - -// UpdateEcosystemStats republishes the per-ecosystem gauges from a fresh snapshot. -// -// Rows are summed by label before anything is published, because normalizing -// collapses aliases: a database holding both "gem" and "rubygems" rows -- the -// proxy writes the former, git-pkgs the latter -- arrives as two rows belonging -// to one label, and publishing them one at a time would leave only the last. -// -// The vectors are not Reset() first. Reset followed by a repopulating loop -// leaves a window in which a scrape sees the families empty or half filled, -// which renders as a spurious gap on any panel built from them. Each series is -// Set instead, and only labels that have actually disappeared are deleted. -func UpdateEcosystemStats(stats []EcosystemStats) { - totals := make(map[string]EcosystemStats, len(stats)) - for _, s := range stats { - ecosystem := purl.NormalizeEcosystem(s.Ecosystem) - t := totals[ecosystem] - t.Packages += s.Packages - t.Versions += s.Versions - t.Artifacts += s.Artifacts - t.CacheSize += s.CacheSize - t.Downloads += s.Downloads - t.DownloadedBytes += s.DownloadedBytes - totals[ecosystem] = t - } - - for ecosystem, t := range totals { - EcosystemDownloadedBytes.WithLabelValues(ecosystem).Set(float64(t.DownloadedBytes)) - EcosystemDownloads.WithLabelValues(ecosystem).Set(float64(t.Downloads)) - EcosystemCacheSize.WithLabelValues(ecosystem).Set(float64(t.CacheSize)) - EcosystemCachedArtifacts.WithLabelValues(ecosystem).Set(float64(t.Artifacts)) - EcosystemPackages.WithLabelValues(ecosystem).Set(float64(t.Packages)) - EcosystemVersions.WithLabelValues(ecosystem).Set(float64(t.Versions)) - } - - publishedMu.Lock() - defer publishedMu.Unlock() - - for ecosystem := range publishedEcosystems { - if _, still := totals[ecosystem]; still { - continue - } - EcosystemDownloadedBytes.DeleteLabelValues(ecosystem) - EcosystemDownloads.DeleteLabelValues(ecosystem) - EcosystemCacheSize.DeleteLabelValues(ecosystem) - EcosystemCachedArtifacts.DeleteLabelValues(ecosystem) - EcosystemPackages.DeleteLabelValues(ecosystem) - EcosystemVersions.DeleteLabelValues(ecosystem) - } - - publishedEcosystems = make(map[string]bool, len(totals)) - for ecosystem := range totals { - publishedEcosystems[ecosystem] = true - } -} - -// MetadataCacheStats is one ecosystem's share of the metadata cache. -type MetadataCacheStats struct { - Ecosystem string - Entries int64 - Size int64 -} - -// UpdateMetadataCacheStats republishes the metadata cache gauges from a fresh -// snapshot. Like UpdateEcosystemStats it sums rows by normalized label, sets -// each series in place and deletes only labels that have disappeared. -func UpdateMetadataCacheStats(stats []MetadataCacheStats) { - totals := make(map[string]MetadataCacheStats, len(stats)) - for _, s := range stats { - ecosystem := purl.NormalizeEcosystem(s.Ecosystem) - t := totals[ecosystem] - t.Entries += s.Entries - t.Size += s.Size - totals[ecosystem] = t - } - - for ecosystem, t := range totals { - MetadataCacheEntries.WithLabelValues(ecosystem).Set(float64(t.Entries)) - MetadataCacheSize.WithLabelValues(ecosystem).Set(float64(t.Size)) - } - - publishedMu.Lock() - defer publishedMu.Unlock() - - for ecosystem := range publishedMetadataEcosystems { - if _, still := totals[ecosystem]; still { - continue - } - MetadataCacheEntries.DeleteLabelValues(ecosystem) - MetadataCacheSize.DeleteLabelValues(ecosystem) - } - - publishedMetadataEcosystems = make(map[string]bool, len(totals)) - for ecosystem := range totals { - publishedMetadataEcosystems[ecosystem] = true - } -} - // UpdateCircuitBreakerState updates circuit breaker state gauge. // state: 0=closed, 1=half-open, 2=open func UpdateCircuitBreakerState(registry string, state int) { @@ -651,11 +229,6 @@ func RecordCircuitBreakerTrip(registry string) { CircuitBreakerTrips.WithLabelValues(registry).Inc() } -// RecordUpstreamRateLimitHold counts a hold placed on host. -func RecordUpstreamRateLimitHold(host string) { - UpstreamRateLimitHolds.WithLabelValues(host).Inc() -} - // IncrementActiveRequests increments the active request counter. func IncrementActiveRequests() { ActiveRequests.Inc() diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 25ae1cd..445a715 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" dto "github.com/prometheus/client_model/go" ) @@ -21,31 +20,28 @@ func TestRecordRequest(t *testing.T) { } func TestRecordCacheOperations(t *testing.T) { - RecordArtifactCacheHit("npm") - RecordArtifactCacheHit("npm") - RecordArtifactCacheMiss("npm") - RecordMetadataCacheHit("pypi", 128) - RecordMetadataCacheMiss("pypi") + RecordCacheHit("npm") + RecordCacheHit("npm") + RecordCacheMiss("npm") + RecordCacheMiss("pypi") // No panics = success } func TestRecordUpstreamOperations(t *testing.T) { - isolateRecentFailures(t) RecordUpstreamFetch("npm", 500*time.Millisecond) RecordUpstreamFetch("pypi", 1*time.Second) - RecordUpstreamError("npm", "fetch_failed", FailureDetail{}) - RecordUpstreamError("pypi", "not_found", FailureDetail{}) + RecordUpstreamError("npm", "fetch_failed") + RecordUpstreamError("pypi", "not_found") // No panics = success } func TestRecordStorageOperations(t *testing.T) { - isolateRecentFailures(t) RecordStorageOperation("read", 10*time.Millisecond) RecordStorageOperation("write", 50*time.Millisecond) - RecordStorageError("read", FailureDetail{}) - RecordStorageError("write", FailureDetail{}) + RecordStorageError("read") + RecordStorageError("write") // No panics = success } @@ -89,7 +85,6 @@ func TestMetricsAreRegistered(t *testing.T) { CachedArtifacts, CircuitBreakerState, CircuitBreakerTrips, - UpstreamRateLimitHolds, ActiveRequests, } @@ -115,9 +110,9 @@ func TestMetricsAreRegistered(t *testing.T) { func TestCacheHitRatio(t *testing.T) { // Reset by recording some hits and misses - RecordArtifactCacheHit("test") - RecordArtifactCacheHit("test") - RecordArtifactCacheMiss("test") + RecordCacheHit("test") + RecordCacheHit("test") + RecordCacheMiss("test") // Collect metrics hits := getMetricValue(t, CacheHits, "test") @@ -180,7 +175,7 @@ func getMetricValue(t *testing.T, collector prometheus.Collector, labelValue str func TestMetricsEndpointOutput(t *testing.T) { // Record some test metrics RecordRequest("npm", 200, 50*time.Millisecond) - RecordArtifactCacheHit("npm") + RecordCacheHit("npm") UpdateCacheStats(1024*1024, 10) // Get the handler and check it produces output @@ -196,120 +191,22 @@ func TestMetricsEndpointOutput(t *testing.T) { func TestMetricsLabeling(t *testing.T) { // Test that different ecosystems are properly labeled - ecosystems := []struct { - input string - label string - }{ - {input: "npm", label: "npm"}, - {input: "pypi", label: "pypi"}, - {input: "cargo", label: "cargo"}, - {input: "gem", label: "rubygems"}, - } + ecosystems := []string{"npm", "pypi", "cargo", "gem"} for _, eco := range ecosystems { - RecordRequest(eco.input, 200, 10*time.Millisecond) - RecordArtifactCacheHit(eco.input) + RecordRequest(eco, 200, 10*time.Millisecond) + RecordCacheHit(eco) } // Verify each ecosystem has metrics for _, eco := range ecosystems { - val := getMetricValue(t, CacheHits, eco.label) + val := getMetricValue(t, CacheHits, eco) if val == 0 { - t.Errorf("no cache hits recorded for %s", eco.label) + t.Errorf("no cache hits recorded for %s", eco) } } } -func TestCacheMetricLabelsAreNormalized(t *testing.T) { - rubyHitsBefore := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems", CacheKindArtifact)) - composerMissesBefore := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist", CacheKindArtifact)) - goMetadataHitsBefore := testutil.ToFloat64(CacheHits.WithLabelValues("golang", CacheKindMetadata)) - goServedBefore := testutil.ToFloat64(MetadataServedBytes.WithLabelValues("golang")) - - RecordArtifactCacheHit("gem") - RecordArtifactCacheMiss("composer") - RecordMetadataCacheHit("go", 10) - - if diff := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems", CacheKindArtifact)) - rubyHitsBefore; diff != 1 { - t.Errorf("rubygems cache hits delta = %.0f, want 1", diff) - } - if diff := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist", CacheKindArtifact)) - composerMissesBefore; diff != 1 { - t.Errorf("packagist cache misses delta = %.0f, want 1", diff) - } - if diff := testutil.ToFloat64(CacheHits.WithLabelValues("golang", CacheKindMetadata)) - goMetadataHitsBefore; diff != 1 { - t.Errorf("golang metadata hits delta = %.0f, want 1", diff) - } - if diff := testutil.ToFloat64(MetadataServedBytes.WithLabelValues("golang")) - goServedBefore; diff != 10 { - t.Errorf("golang metadata served bytes delta = %.0f, want 10", diff) - } -} - -// TestCacheKindsAreCountedApart checks an artifact hit and a metadata hit land -// in different series. Only metadata hits add served bytes. -func TestCacheKindsAreCountedApart(t *testing.T) { - const eco = "kind-test" - artifactHits := func() float64 { return testutil.ToFloat64(CacheHits.WithLabelValues(eco, CacheKindArtifact)) } - metadataHits := func() float64 { return testutil.ToFloat64(CacheHits.WithLabelValues(eco, CacheKindMetadata)) } - artifactMisses := func() float64 { return testutil.ToFloat64(CacheMisses.WithLabelValues(eco, CacheKindArtifact)) } - metadataMisses := func() float64 { return testutil.ToFloat64(CacheMisses.WithLabelValues(eco, CacheKindMetadata)) } - served := func() float64 { return testutil.ToFloat64(MetadataServedBytes.WithLabelValues(eco)) } - - aHits, mHits, aMisses, mMisses, bytes := artifactHits(), metadataHits(), artifactMisses(), metadataMisses(), served() - - RecordArtifactCacheHit(eco) - RecordMetadataCacheHit(eco, 300) - RecordMetadataCacheHit(eco, 0) - RecordMetadataCacheMiss(eco) - - checks := []struct { - name string - got, want float64 - }{ - {"artifact hits", artifactHits() - aHits, 1}, - {"metadata hits", metadataHits() - mHits, 2}, - {"artifact misses", artifactMisses() - aMisses, 0}, - {"metadata misses", metadataMisses() - mMisses, 1}, - {"metadata served bytes", served() - bytes, 300}, - } - for _, c := range checks { - if c.got != c.want { - t.Errorf("%s delta = %.0f, want %.0f", c.name, c.got, c.want) - } - } -} - -func TestUpdateMetadataCacheStats(t *testing.T) { - UpdateMetadataCacheStats([]MetadataCacheStats{ - {Ecosystem: "gem", Entries: 2, Size: 100}, - {Ecosystem: "rubygems", Entries: 3, Size: 50}, - {Ecosystem: "npm", Entries: 1, Size: 10}, - }) - - if got := testutil.ToFloat64(MetadataCacheEntries.WithLabelValues("rubygems")); got != 5 { - t.Errorf("rubygems entries = %.0f, want 5 (aliases summed)", got) - } - if got := testutil.ToFloat64(MetadataCacheSize.WithLabelValues("rubygems")); got != 150 { - t.Errorf("rubygems size = %.0f, want 150 (aliases summed)", got) - } - if got := testutil.CollectAndCount(MetadataCacheSize); got != 2 { - t.Errorf("size series = %d, want 2", got) - } - - UpdateMetadataCacheStats([]MetadataCacheStats{{Ecosystem: "npm", Entries: 4, Size: 40}}) - - if got := testutil.CollectAndCount(MetadataCacheSize); got != 1 { - t.Errorf("size series after rubygems disappeared = %d, want 1", got) - } - if got := testutil.CollectAndCount(MetadataCacheEntries); got != 1 { - t.Errorf("entry series after rubygems disappeared = %d, want 1", got) - } - if got := testutil.ToFloat64(MetadataCacheSize.WithLabelValues("npm")); got != 40 { - t.Errorf("npm size = %.0f, want 40", got) - } - - UpdateMetadataCacheStats(nil) -} - func TestMetricNames(t *testing.T) { // Verify metric names follow Prometheus naming conventions expectedMetrics := []string{ @@ -317,9 +214,6 @@ func TestMetricNames(t *testing.T) { "proxy_request_duration_seconds", "proxy_cache_hits_total", "proxy_cache_misses_total", - "proxy_metadata_cache_served_bytes_total", - "proxy_metadata_cache_size_bytes", - "proxy_metadata_cache_entries", "proxy_upstream_fetch_duration_seconds", "proxy_upstream_errors_total", "proxy_storage_operation_duration_seconds", diff --git a/internal/metrics/snapshot.go b/internal/metrics/snapshot.go deleted file mode 100644 index 2c96dff..0000000 --- a/internal/metrics/snapshot.go +++ /dev/null @@ -1,165 +0,0 @@ -package metrics - -import ( - "fmt" - "sort" - - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" -) - -// Sample is one time series read out of the registry at a point in time. -// -// Counters and gauges carry their value in Value. Histograms carry their -// observation count and total in Count and Sum, and leave Value at zero — -// there is no single "value" for a histogram, and the UI shows a mean derived -// from Sum/Count rather than pretending otherwise. -type Sample struct { - Labels map[string]string - Value float64 - Count uint64 - Sum float64 -} - -// Label returns the value of one label, or "" if absent. -func (s Sample) Label(name string) string { - return s.Labels[name] -} - -// Mean returns the average observation of a histogram sample, or 0 when -// nothing has been observed yet. -func (s Sample) Mean() float64 { - if s.Count == 0 { - return 0 - } - return s.Sum / float64(s.Count) -} - -// Snapshot is the value of every registered proxy metric at one instant. -// -// These are process-lifetime figures held in the Prometheus registry, not -// database state: they start at zero when the proxy starts and are lost on -// restart. Anything that needs to survive a restart, or needs history, has to -// come from a Prometheus server scraping /metrics. -type Snapshot struct { - families map[string][]Sample -} - -// Gather reads the default Prometheus registry. It is the same data /metrics -// serves, shaped for rendering rather than for scraping. -func Gather() (*Snapshot, error) { - return GatherFrom(prometheus.DefaultGatherer) -} - -// GatherFrom reads an explicit gatherer, so tests can supply their own registry. -func GatherFrom(g prometheus.Gatherer) (*Snapshot, error) { - families, err := g.Gather() - if err != nil { - return nil, fmt.Errorf("gathering metrics: %w", err) - } - - snap := &Snapshot{families: make(map[string][]Sample, len(families))} - for _, mf := range families { - samples := make([]Sample, 0, len(mf.GetMetric())) - for _, m := range mf.GetMetric() { - samples = append(samples, sampleOf(m)) - } - snap.families[mf.GetName()] = samples - } - return snap, nil -} - -func sampleOf(m *dto.Metric) Sample { - s := Sample{Labels: make(map[string]string, len(m.GetLabel()))} - for _, lp := range m.GetLabel() { - s.Labels[lp.GetName()] = lp.GetValue() - } - - switch { - case m.GetCounter() != nil: - s.Value = m.GetCounter().GetValue() - case m.GetGauge() != nil: - s.Value = m.GetGauge().GetValue() - case m.GetHistogram() != nil: - h := m.GetHistogram() - s.Count = h.GetSampleCount() - s.Sum = h.GetSampleSum() - case m.GetSummary() != nil: - sm := m.GetSummary() - s.Count = sm.GetSampleCount() - s.Sum = sm.GetSampleSum() - case m.GetUntyped() != nil: - s.Value = m.GetUntyped().GetValue() - } - return s -} - -// Samples returns every series of a metric, or nil when it has never been -// observed. Series are ordered by their label values so that rendering is -// stable between scrapes. -func (s *Snapshot) Samples(name string) []Sample { - if s == nil { - return nil - } - out := append([]Sample(nil), s.families[name]...) - sort.Slice(out, func(i, j int) bool { - return labelKey(out[i].Labels) < labelKey(out[j].Labels) - }) - return out -} - -func labelKey(labels map[string]string) string { - names := make([]string, 0, len(labels)) - for n := range labels { - names = append(names, n) - } - sort.Strings(names) - - key := "" - for _, n := range names { - key += n + "=" + labels[n] + "," - } - return key -} - -// Sum totals every series of a counter or gauge. -func (s *Snapshot) Sum(name string) float64 { - var total float64 - for _, sample := range s.Samples(name) { - total += sample.Value - } - return total -} - -// Count adds up the observation counts of a histogram across every series. -func (s *Snapshot) Count(name string) uint64 { - var total uint64 - for _, sample := range s.Samples(name) { - total += sample.Count - } - return total -} - -// Mean returns the average observation of a histogram across every series, or -// 0 when nothing has been observed. -func (s *Snapshot) Mean(name string) float64 { - var count uint64 - var sum float64 - for _, sample := range s.Samples(name) { - count += sample.Count - sum += sample.Sum - } - if count == 0 { - return 0 - } - return sum / float64(count) -} - -// SumBy groups a metric by one label and totals each group. -func (s *Snapshot) SumBy(name, label string) map[string]float64 { - out := make(map[string]float64) - for _, sample := range s.Samples(name) { - out[sample.Label(label)] += sample.Value - } - return out -} diff --git a/internal/metrics/snapshot_test.go b/internal/metrics/snapshot_test.go deleted file mode 100644 index 5b6c7fd..0000000 --- a/internal/metrics/snapshot_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package metrics - -import ( - "testing" - - "github.com/prometheus/client_golang/prometheus" -) - -// newTestRegistry builds an isolated registry so these tests are not affected -// by counters other tests in this package have already incremented. -func newTestRegistry(t *testing.T) (*prometheus.Registry, *prometheus.CounterVec, *prometheus.HistogramVec, *prometheus.GaugeVec) { - t.Helper() - - reg := prometheus.NewRegistry() - counter := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "test_errors_total", Help: "t"}, - []string{"ecosystem", "kind"}, - ) - hist := prometheus.NewHistogramVec( - prometheus.HistogramOpts{Name: "test_duration_seconds", Help: "t"}, - []string{"op"}, - ) - gauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_active", Help: "t"}, []string{}) - reg.MustRegister(counter, hist, gauge) - return reg, counter, hist, gauge -} - -func TestSnapshotSumAndSumBy(t *testing.T) { - reg, counter, _, gauge := newTestRegistry(t) - counter.WithLabelValues("npm", "timeout").Add(3) - counter.WithLabelValues("npm", "refused").Add(2) - counter.WithLabelValues("pypi", "timeout").Add(5) - gauge.WithLabelValues().Set(7) - - snap, err := GatherFrom(reg) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - - if got := snap.Sum("test_errors_total"); got != 10 { - t.Errorf("Sum = %v, want 10", got) - } - if got := snap.Sum("test_active"); got != 7 { - t.Errorf("gauge Sum = %v, want 7", got) - } - - byEco := snap.SumBy("test_errors_total", "ecosystem") - if byEco["npm"] != 5 || byEco["pypi"] != 5 { - t.Errorf("SumBy(ecosystem) = %v, want npm=5 pypi=5", byEco) - } -} - -func TestSnapshotHistogram(t *testing.T) { - reg, _, hist, _ := newTestRegistry(t) - hist.WithLabelValues("get").Observe(0.010) - hist.WithLabelValues("get").Observe(0.030) - hist.WithLabelValues("put").Observe(0.100) - - snap, err := GatherFrom(reg) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - - if got := snap.Count("test_duration_seconds"); got != 3 { - t.Errorf("Count = %d, want 3", got) - } - // (0.010 + 0.030 + 0.100) / 3 - if got := snap.Mean("test_duration_seconds"); got < 0.0466 || got > 0.0467 { - t.Errorf("Mean = %v, want ~0.04667", got) - } - - // A histogram carries no single value, so Value stays zero. - for _, s := range snap.Samples("test_duration_seconds") { - if s.Value != 0 { - t.Errorf("histogram sample carries Value %v, want 0", s.Value) - } - if s.Label("op") == "get" && s.Count != 2 { - t.Errorf("get count = %d, want 2", s.Count) - } - } -} - -func TestSnapshotSamplesAreOrdered(t *testing.T) { - reg, counter, _, _ := newTestRegistry(t) - for _, eco := range []string{"pypi", "npm", "cargo"} { - counter.WithLabelValues(eco, "timeout").Inc() - } - - snap, err := GatherFrom(reg) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - - var got []string - for _, s := range snap.Samples("test_errors_total") { - got = append(got, s.Label("ecosystem")) - } - want := []string{"cargo", "npm", "pypi"} - for i := range want { - if got[i] != want[i] { - t.Fatalf("sample order = %v, want %v (stable rendering depends on it)", got, want) - } - } -} - -func TestSnapshotMissingMetric(t *testing.T) { - reg, _, _, _ := newTestRegistry(t) - snap, err := GatherFrom(reg) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - - if got := snap.Samples("nope_total"); got != nil { - t.Errorf("Samples of an unknown metric = %v, want nil", got) - } - if got := snap.Sum("nope_total"); got != 0 { - t.Errorf("Sum of an unknown metric = %v, want 0", got) - } - if got := snap.Mean("nope_total"); got != 0 { - t.Errorf("Mean of an unknown metric = %v, want 0", got) - } -} - -// Gather reads the real registry, so every metric this package registers must -// come back. This is the check that a newly added metric is reachable by the UI. -func TestGatherSeesRegisteredMetrics(t *testing.T) { - isolateRecentFailures(t) - RecordArtifactCacheHit("npm") - RecordScanError("npm", "clamav", "timeout", FailureDetail{}) - - snap, err := Gather() - if err != nil { - t.Fatalf("Gather: %v", err) - } - - for _, name := range []string{"proxy_cache_hits_total", "proxy_scan_errors_total"} { - if len(snap.Samples(name)) == 0 { - t.Errorf("%s is registered but absent from the snapshot", name) - } - } -} diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go deleted file mode 100644 index 30a9771..0000000 --- a/internal/mirror/apk.go +++ /dev/null @@ -1,154 +0,0 @@ -package mirror - -import ( - "context" - "errors" - "fmt" - "regexp" - "strings" - "sync" - "time" - - "github.com/git-pkgs/proxy/internal/handler" - "github.com/git-pkgs/registries" -) - -// apkIndexTTL is how long a repository's APKINDEX is reused, so mirroring -// the packages of an image reads each index once. -const apkIndexTTL = 5 * time.Minute - -var ( - // apkDistroPattern takes the release from syft's distro qualifier, such - // as alpine-3.20.3 for v3.20. Edge, as alpine-3.21.0_alpha20240807, - // isn't a release and doesn't match. - apkDistroPattern = regexp.MustCompile(`^alpine-(\d+)\.(\d+)(?:\.\d+)?$`) - apkArchPattern = regexp.MustCompile(`^[a-z0-9_]+$`) - // apkComponents are the repositories of a release a package is looked - // up in, in order. - apkComponents = []string{"main", "community"} -) - -// apkIndexes keeps one registries client per repository directory, which -// holds the parsed APKINDEX. -type apkIndexes struct { - mu sync.Mutex - regs map[string]apkIndex -} - -type apkIndex struct { - reg registries.Registry - fetched time.Time -} - -// mirrorAPK mirrors an Alpine package under the path the handler caches it -// at, {repository}/v{release}/{component}/{arch}/{file}. The PURL namespace -// names the upstream.apk repository; the distro and arch qualifiers give -// the release and architecture, and the package is looked up in the main -// and community indexes of that release. -func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - fail := func(err error) { m.recordFailure(tracker, pv, err) } - repository, name := pv.Namespace, pv.Name - if repository == "" { - repository = "alpine" - } - repositoryURL, ok := handler.APKRepositories(m.APKRepositories)[repository] - if !ok { - fail(fmt.Errorf("unknown APK repository %q: it isn't in upstream.apk", repository)) - return - } - match := apkDistroPattern.FindStringSubmatch(pv.Qualifiers["distro"]) - if match == nil { - fail(fmt.Errorf("apk packages need a distro qualifier with the Alpine release, such as distro=alpine-3.20.3, not %q", pv.Qualifiers["distro"])) - return - } - release := "v" + match[1] + "." + match[2] - arch := pv.Qualifiers["arch"] - if !apkArchPattern.MatchString(arch) { - fail(fmt.Errorf("apk packages need an arch qualifier, such as arch=x86_64, not %q", arch)) - return - } - - for _, component := range apkComponents { - dir := release + "/" + component + "/" + arch - artifact, found, err := m.apkArtifact(ctx, repositoryURL+"/"+dir, name, pv.Version) - if err != nil { - fail(fmt.Errorf("reading %s %s/APKINDEX.tar.gz: %w", repository, dir, err)) - return - } - if !found { - continue - } - cacheFilename := handler.APKCacheFilename(repository, dir+"/"+artifact.Filename) - result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, pv.Ecosystem, name, pv.Version, cacheFilename, artifact.URL) - if err != nil { - fail(err) - return - } - m.recordResult(tracker, pv, result) - return - } - fail(fmt.Errorf("%s %s isn't in %s of %s %s for %s", name, pv.Version, strings.Join(apkComponents, " or "), repository, release, arch)) -} - -// apkArtifact returns the file of version of name that the index in dirURL -// lists, and whether it lists one. -func (m *Mirror) apkArtifact(ctx context.Context, dirURL, name, version string) (registries.Artifact, bool, error) { - reg, err := m.apkRegistry(dirURL) - if err != nil { - return registries.Artifact{}, false, err - } - versions, err := reg.FetchVersions(ctx, name) - if isNotFound(err) { - return registries.Artifact{}, false, nil - } - if err != nil { - m.apk.forget(dirURL) - return registries.Artifact{}, false, err - } - for _, v := range versions { - if v.Number == version && len(v.Artifacts) > 0 { - return v.Artifacts[0], true, nil - } - } - return registries.Artifact{}, false, nil -} - -// apkRegistry returns the client for the repository directory at dirURL, -// reusing it, and the index it read, for apkIndexTTL. -func (m *Mirror) apkRegistry(dirURL string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface - m.apk.mu.Lock() - defer m.apk.mu.Unlock() - if idx, ok := m.apk.regs[dirURL]; ok && time.Since(idx.fetched) < apkIndexTTL { - return idx.reg, nil - } - r, _ := m.Versions.(RegistryVersions) - // Naming the v2 index reads it also where Packages.adb is served beside - // it, which the client would try first and refuse. - reg, err := registries.New("apk", dirURL+"/APKINDEX.tar.gz", r.client()) - if err != nil { - return nil, err - } - if m.apk.regs == nil { - m.apk.regs = map[string]apkIndex{} - } - m.apk.regs[dirURL] = apkIndex{reg: reg, fetched: time.Now()} - return reg, nil -} - -// forget drops the client for dirURL, so an index that failed to load is -// read again next time. -func (a *apkIndexes) forget(dirURL string) { - a.mu.Lock() - defer a.mu.Unlock() - delete(a.regs, dirURL) -} - -// isNotFound reports whether err means a package or index doesn't exist, -// such as a release without a community repository for an architecture. -func isNotFound(err error) bool { - if errors.Is(err, registries.ErrNotFound) { - return true - } - var httpErr *registries.HTTPError - return errors.As(err, &httpErr) && httpErr.IsNotFound() -} diff --git a/internal/mirror/brew_test.go b/internal/mirror/brew_test.go deleted file mode 100644 index 80e5a26..0000000 --- a/internal/mirror/brew_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package mirror - -import ( - "context" - "errors" - "strings" - "sync" - "testing" - - "github.com/git-pkgs/proxy/internal/handler" -) - -// fakeBottles resolves every formula version to homebrew/core/ tagged -// -1, like a formula whose bottles were rebuilt once. -type fakeBottles struct { - err error -} - -func (f fakeBottles) BottleImage(_ context.Context, formula, version string) (handler.ImageRef, error) { - if f.err != nil { - return handler.ImageRef{}, f.err - } - return handler.ImageRef{Name: "homebrew/core/" + formula, References: []string{version + "-1"}}, nil -} - -// platformImages records the images mirrored and the platforms asked for. -type platformImages struct { - mu sync.Mutex - refs []handler.ImageRef - platforms [][]handler.Platform -} - -func (p *platformImages) MirrorImage(_ context.Context, ref handler.ImageRef, platforms []handler.Platform) (handler.ImageResult, error) { - p.mu.Lock() - defer p.mu.Unlock() - p.refs = append(p.refs, ref) - p.platforms = append(p.platforms, platforms) - return handler.ImageResult{Fetched: 2, Bytes: 10}, nil -} - -type stubVersions map[string][]string - -func (s stubVersions) ListVersions(_ context.Context, ecosystem, name string) ([]string, error) { - return s[ecosystem+"/"+name], nil -} - -func TestMirrorBrewMirrorsBottleImage(t *testing.T) { - m := setupTestMirror(t, 1) - images := &platformImages{} - m.Images, m.Bottles = images, fakeBottles{} - - source := &PURLSource{ - PURLs: []string{"pkg:brew/hello@2.12.3", "pkg:brew/jq"}, - Platforms: []string{"linux/arm64"}, - Versions: stubVersions{"brew/jq": {"1.8.1"}}, - } - progress, err := m.Run(context.Background(), source) - if err != nil { - t.Fatal(err) - } - if progress.Completed != 2 || progress.Failed != 0 { - t.Fatalf("progress = %+v, errors %v", progress, progress.Errors) - } - got := map[string]string{} - for i, ref := range images.refs { - if len(images.platforms[i]) != 1 || images.platforms[i][0].OS != "linux" || images.platforms[i][0].Architecture != "arm64" { - t.Errorf("%s: platforms = %+v, want linux/arm64", ref.Name, images.platforms[i]) - } - got[ref.Name] = strings.Join(ref.References, ",") - } - if got["homebrew/core/hello"] != "2.12.3-1" || got["homebrew/core/jq"] != "1.8.1-1" { - t.Errorf("images = %v", got) - } -} - -func TestMirrorBrewFailures(t *testing.T) { - m := setupTestMirror(t, 1) - progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:brew/hello@2.12.3"}}) - if err != nil { - t.Fatal(err) - } - if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "isn't available") { - t.Errorf("without a bottle resolver: %+v", progress) - } - - images := &platformImages{} - m.Images, m.Bottles = images, fakeBottles{err: errors.New(`homebrew/core has no formula "nope"`)} - progress, err = m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:brew/nope@1.0"}}) - if err != nil { - t.Fatal(err) - } - if progress.Failed != 1 || len(images.refs) != 0 { - t.Errorf("unknown formula: %+v, mirrored %v", progress, images.refs) - } -} diff --git a/internal/mirror/cache_test.go b/internal/mirror/cache_test.go deleted file mode 100644 index 661e408..0000000 --- a/internal/mirror/cache_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package mirror - -import ( - "context" - "errors" - "io" - "net/http" - "net/http/httptest" - "sync/atomic" - "testing" - - "github.com/git-pkgs/registries" - "github.com/git-pkgs/registries/client" - "github.com/git-pkgs/registries/fetch" -) - -type mirrorCacheRegistry struct { - downloadURL string - resolutions atomic.Int64 -} - -type rejectingMirrorTransport struct{} - -func (rejectingMirrorTransport) RoundTrip(*http.Request) (*http.Response, error) { - return nil, errors.New("unexpected artifact download on a cache hit") -} - -func (*mirrorCacheRegistry) Ecosystem() string { return "npm" } -func (*mirrorCacheRegistry) FetchVersions(context.Context, string) ([]registries.Version, error) { - return nil, errors.New("unexpected metadata request") -} -func (r *mirrorCacheRegistry) URLs() client.URLBuilder { //nolint:ireturn // required by fetch.Registry - return &client.BaseURLs{DownloadFn: func(_, _ string) string { - r.resolutions.Add(1) - return r.downloadURL - }} -} - -func TestMirrorRepeatedRunSkipsCachedArtifact(t *testing.T) { - var downloads atomic.Int64 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/example-1.0.0.tgz" { - t.Errorf("unexpected artifact path: %s", r.URL.Path) - http.NotFound(w, r) - return - } - downloads.Add(1) - _, _ = io.WriteString(w, "tarball bytes") - })) - defer upstream.Close() - m := setupTestMirror(t, 1) - registry := &mirrorCacheRegistry{downloadURL: upstream.URL + "/example-1.0.0.tgz"} - m.proxy.Resolver.RegisterRegistry(registry) - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - t.Cleanup(func() { _ = fetcher.Close() }) - m.proxy.Fetcher = fetcher - source := &PURLSource{PURLs: []string{"pkg:npm/example@1.0.0"}} - first, err := m.Run(context.Background(), source) - if err != nil || first.Completed != 1 || first.Skipped != 0 || first.Failed != 0 { - t.Fatalf("first mirror: progress=%+v err=%v", first, err) - } - before, err := m.db.GetArtifact("pkg:npm/example@1.0.0", "example-1.0.0.tgz") - if err != nil { - t.Fatal(err) - } - // The second run must succeed even when the artifact server is offline. - upstream.Close() - second, err := m.Run(context.Background(), source) - if err != nil || second.Completed != 0 || second.Skipped != 1 || second.Failed != 0 { - t.Fatalf("second mirror: progress=%+v err=%v", second, err) - } - after, err := m.db.GetArtifact("pkg:npm/example@1.0.0", "example-1.0.0.tgz") - if err != nil { - t.Fatal(err) - } - if before.FetchedAt != after.FetchedAt || before.ContentHash != after.ContentHash || before.StoragePath != after.StoragePath { - t.Fatal("second mirror changed the cached artifact") - } - if downloads.Load() != 1 || registry.resolutions.Load() != 2 { - t.Fatalf("downloads=%d resolutions=%d; want 1 and 2", downloads.Load(), registry.resolutions.Load()) - } -} diff --git a/internal/mirror/cran.go b/internal/mirror/cran.go deleted file mode 100644 index 02d26bd..0000000 --- a/internal/mirror/cran.go +++ /dev/null @@ -1,191 +0,0 @@ -package mirror - -import ( - "bufio" - "bytes" - "compress/gzip" - "context" - "fmt" - "io" - "net/url" - "regexp" - "strings" - "sync" - "time" - - "github.com/git-pkgs/registries" -) - -const ( - cranDefaultURL = "https://cloud.r-project.org" - // cranIndexTTL is how long a fetched src/contrib/PACKAGES is reused, so - // mirroring many packages, or every version of one, reads it once. - cranIndexTTL = 5 * time.Minute - // cranMaxIndexSize limits the decompressed PACKAGES. - // CRAN's PACKAGES is about 7 MB. - cranMaxIndexSize = 64 << 20 - // cranMaxLine limits a line of PACKAGES. - cranMaxLine = 1 << 20 -) - -var ( - cranNamePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9.]*$`) - cranVersionPattern = regexp.MustCompile(`^[0-9]+[0-9.-]*$`) -) - -// cranIndex caches the MD5 sums src/contrib/PACKAGES lists for the current -// source packages of a CRAN repository, by file name. -type cranIndex struct { - mu sync.Mutex - baseURL string - fetched time.Time - md5s map[string]string -} - -// mirrorCRAN mirrors a CRAN source package under ("cran", name, version, -// name_version.tar.gz), the one entry the handler serves for both -// src/contrib and src/contrib/Archive. -func (m *Mirror) mirrorCRAN(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - fail := func(err error) { m.recordFailure(tracker, pv, err) } - if !cranNamePattern.MatchString(pv.Name) { - fail(fmt.Errorf("invalid CRAN package name %q", pv.Name)) - return - } - if !cranVersionPattern.MatchString(pv.Version) { - fail(fmt.Errorf("invalid CRAN version %q", pv.Version)) - return - } - baseURL, client, err := m.cranRepository(pv.Name) - if err != nil { - fail(err) - return - } - filename := pv.Name + "_" + pv.Version + ".tar.gz" - - current, err := m.cranCurrentMD5s(ctx, baseURL, client) - if err != nil { - fail(fmt.Errorf("reading src/contrib/PACKAGES: %w", err)) - return - } - downloadURL := baseURL + "/src/contrib/" + filename - sum, ok := current[filename] - if !ok { - downloadURL = baseURL + "/src/contrib/Archive/" + url.PathEscape(pv.Name) + "/" + filename - } - var declared string - if sum != "" { - declared = "md5-" + sum - } - result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, pv.Name, pv.Version, filename, downloadURL, declared) - if err != nil { - fail(err) - return - } - m.recordResult(tracker, pv, result) -} - -// cranRepository returns the repository to mirror a CRAN package from and -// the client to read its indexes with: those Versions lists versions with, -// which come from upstream.cran, or the public CRAN. -func (m *Mirror) cranRepository(name string) (string, *registries.Client, error) { - r, _ := m.Versions.(RegistryVersions) - baseURL := cranDefaultURL - if r.URL != nil { - configured, err := r.URL("cran", name) - if err != nil { - return "", nil, err - } - if configured != "" { - baseURL = strings.TrimSuffix(configured, "/") - } - } - client := r.Client - if client == nil { - client = registries.DefaultClient() - } - return baseURL, client, nil -} - -// cranCurrentMD5s returns the MD5 sums of the current source packages of the -// repository at baseURL, reading src/contrib/PACKAGES.gz unless it was read -// within cranIndexTTL. -func (m *Mirror) cranCurrentMD5s(ctx context.Context, baseURL string, client *registries.Client) (map[string]string, error) { - m.cran.mu.Lock() - defer m.cran.mu.Unlock() - if m.cran.md5s != nil && m.cran.baseURL == baseURL && time.Since(m.cran.fetched) < cranIndexTTL { - return m.cran.md5s, nil - } - body, err := client.GetBody(ctx, baseURL+"/src/contrib/PACKAGES.gz") - if err != nil { - return nil, err - } - data, err := gunzipLimited(body) - if err != nil { - return nil, err - } - sums, err := parseCRANPackages(data) - if err != nil { - return nil, err - } - m.cran.md5s = sums - m.cran.baseURL = baseURL - m.cran.fetched = time.Now() - return m.cran.md5s, nil -} - -// gunzipLimited decompresses data, or returns it as is when it isn't -// gzipped. -func gunzipLimited(data []byte) ([]byte, error) { - if !bytes.HasPrefix(data, []byte{0x1f, 0x8b}) { - return data, nil - } - zr, err := gzip.NewReader(bytes.NewReader(data)) - if err != nil { - return nil, err - } - defer func() { _ = zr.Close() }() - out, err := io.ReadAll(io.LimitReader(zr, cranMaxIndexSize+1)) - if err != nil { - return nil, err - } - if len(out) > cranMaxIndexSize { - return nil, fmt.Errorf("index is larger than %d bytes", cranMaxIndexSize) - } - return out, nil -} - -// parseCRANPackages returns the MD5sum of each source package a PACKAGES -// file lists, by file name. Entries without one are listed with "". -func parseCRANPackages(data []byte) (map[string]string, error) { - sums := map[string]string{} - var name, version, sum string - flush := func() { - if name != "" && version != "" { - sums[name+"_"+version+".tar.gz"] = sum - } - name, version, sum = "", "", "" - } - scanner := bufio.NewScanner(bytes.NewReader(data)) - scanner.Buffer(nil, cranMaxLine) - for scanner.Scan() { - line := scanner.Text() - if strings.TrimSpace(line) == "" { - flush() - continue - } - key, value, ok := strings.Cut(line, ":") - if !ok || strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") { - continue - } - switch value = strings.TrimSpace(value); key { - case "Package": - name = value - case "Version": - version = value - case "MD5sum": - sum = value - } - } - flush() - return sums, scanner.Err() -} diff --git a/internal/mirror/cran_test.go b/internal/mirror/cran_test.go deleted file mode 100644 index 3eae797..0000000 --- a/internal/mirror/cran_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package mirror - -import ( - "maps" - "testing" -) - -func TestParseCRANPackages(t *testing.T) { - data := []byte(`Package: A3 -Version: 1.0.0 -Depends: R (>= 2.15.0), xtable, - pbapply -MD5sum: 027ebdd8affce8f0effaecfcd5f5ade2 -NeedsCompilation: no - -Package: abc -Version: 2.2.1 -NeedsCompilation: no -`) - want := map[string]string{ - "A3_1.0.0.tar.gz": "027ebdd8affce8f0effaecfcd5f5ade2", - "abc_2.2.1.tar.gz": "", - } - if got, err := parseCRANPackages(data); err != nil || !maps.Equal(got, want) { - t.Errorf("parseCRANPackages = %v, %v, want %v", got, err, want) - } -} diff --git a/internal/mirror/denylist_test.go b/internal/mirror/denylist_test.go deleted file mode 100644 index 5676e87..0000000 --- a/internal/mirror/denylist_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package mirror - -import ( - "context" - "io" - "log/slog" - "strings" - "testing" - - "github.com/git-pkgs/proxy/internal/denylist" - "github.com/git-pkgs/proxy/internal/handler" -) - -func TestMirrorOneDenylist(t *testing.T) { - policy, err := denylist.New([]string{"pkg:npm/demo@1.0.0"}) - if err != nil { - t.Fatal(err) - } - // No database, storage, or fetcher is needed: the denial comes first. - p := &handler.Proxy{Denylist: policy} - m := New(p, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)), 1) - tracker := newProgressTracker() - m.mirrorOne(context.Background(), PackageVersion{Ecosystem: "npm", Name: "demo", Version: "1.0.0"}, tracker) - progress := tracker.snapshot() - if progress.Failed != 1 || progress.Completed != 0 || progress.Skipped != 0 || len(progress.Errors) != 1 { - t.Fatalf("progress = %+v", progress) - } - if !strings.Contains(progress.Errors[0].Error, "denylist") { - t.Fatalf("unexpected error: %s", progress.Errors[0].Error) - } -} diff --git a/internal/mirror/files.go b/internal/mirror/files.go deleted file mode 100644 index 7486e8c..0000000 --- a/internal/mirror/files.go +++ /dev/null @@ -1,118 +0,0 @@ -package mirror - -import ( - "fmt" - "slices" - "strings" - - "github.com/git-pkgs/proxy/internal/handler" - "github.com/git-pkgs/registries" -) - -// fileEcosystem describes an ecosystem whose versions publish several files, -// each mirrored on its own. -type fileEcosystem struct { - // qualifiers are the PURL qualifiers that select files. A file is - // mirrored when each one given matches the value attrs reports for it. - qualifiers []string - attrs func(file registries.Artifact) map[string]string - // cacheKey returns the name, version and filename the download handler - // caches file under, or false when the handler would not cache it. - cacheKey func(pv PackageVersion, file registries.Artifact) (name, version, filename string, ok bool) - // check refuses a package the ecosystem can't mirror, before its files - // are listed. - check func(pv PackageVersion) error -} - -var fileEcosystems = map[string]fileEcosystem{ - "pypi": { - qualifiers: []string{"file_name"}, - attrs: func(file registries.Artifact) map[string]string { - return map[string]string{"file_name": file.Filename} - }, - cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) { - name, version := handler.PyPIFilenameIdentity(file.Filename) - if name == "" { - name, version = pv.Name, pv.Version - } - return name, version, file.Filename, true - }, - }, - "conda": { - qualifiers: []string{"subdir", "build", "type"}, - attrs: func(file registries.Artifact) map[string]string { - subdir, _ := file.Metadata["subdir"].(string) - build, _ := file.Metadata["build"].(string) - return map[string]string{"subdir": subdir, "build": build, "type": condaFileType(file.Filename)} - }, - cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) { - channel, _, _ := strings.Cut(pv.Name, "/") - subdir, _ := file.Metadata["subdir"].(string) - name, version := handler.CondaFilenameIdentity(file.Filename) - if name == "" || subdir == "" || condaFileType(file.Filename) == "" { - return "", "", "", false - } - return channel + "/" + name, version, subdir + "/" + file.Filename, true - }, - check: func(pv PackageVersion) error { - if !strings.Contains(pv.Name, "/") { - return fmt.Errorf("conda packages need a channel, such as pkg:conda/conda-forge/%s", pv.Name) - } - return nil - }, - }, -} - -// condaFileType returns the PURL type qualifier of a conda package file. -func condaFileType(filename string) string { - switch { - case strings.HasSuffix(filename, ".conda"): - return "conda" - case strings.HasSuffix(filename, ".tar.bz2"): - return "tar.bz2" - default: - return "" - } -} - -// selection returns the qualifiers of pv that select files, as -// "key=value" pairs, for error messages. -func (e fileEcosystem) selection(pv PackageVersion) string { - var pairs []string - for _, key := range e.qualifiers { - if value := pv.Qualifiers[key]; value != "" { - pairs = append(pairs, key+"="+value) - } - } - return strings.Join(pairs, " ") -} - -// selects reports whether file matches every selecting qualifier of pv. -func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool { - attrs := e.attrs(file) - for _, key := range e.qualifiers { - if want := pv.Qualifiers[key]; want != "" && attrs[key] != want { - return false - } - } - return true -} - -// fileQualifiers keeps the qualifiers of a PURL that select files of its -// ecosystem, or returns nil. -func fileQualifiers(ecosystem string, qualifiers map[string]string) map[string]string { - e, ok := fileEcosystems[ecosystem] - if !ok { - return nil - } - var kept map[string]string - for key, value := range qualifiers { - if value != "" && slices.Contains(e.qualifiers, key) { - if kept == nil { - kept = map[string]string{} - } - kept[key] = value - } - } - return kept -} diff --git a/internal/mirror/files_test.go b/internal/mirror/files_test.go deleted file mode 100644 index d84af72..0000000 --- a/internal/mirror/files_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package mirror - -import ( - "reflect" - "testing" -) - -func TestPURLSourceCondaChannelAndQualifiers(t *testing.T) { - versions := &fakeVersions{versions: []string{"1.0", "1.1"}} - source := &PURLSource{ - PURLs: []string{ - "pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&build=py312h_0&type=conda", - "pkg:conda/numpy@2.1.0?channel=conda-forge&repository_url=https://example.com", - "pkg:conda/numpy@2.1.0", - "pkg:conda/numpy", - "pkg:conda/numpy?channel=bioconda&subdir=noarch", - }, - Versions: versions, - } - - got := enumerate(t, source) - - want := []PackageVersion{ - {Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0", - Qualifiers: map[string]string{"subdir": "linux-64", "build": "py312h_0", "type": "conda"}}, - {Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0"}, - {Ecosystem: "conda", Name: "numpy", Version: "2.1.0"}, - {Ecosystem: "conda", Name: "numpy"}, - {Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.0", Qualifiers: map[string]string{"subdir": "noarch"}}, - {Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.1", Qualifiers: map[string]string{"subdir": "noarch"}}, - } - if !reflect.DeepEqual(got, want) { - t.Errorf("items = %+v\nwant %+v", got, want) - } - if !reflect.DeepEqual(versions.asked, []string{"conda/bioconda/numpy"}) { - t.Errorf("listed versions of %v, want only the package with a channel", versions.asked) - } -} diff --git a/internal/mirror/git.go b/internal/mirror/git.go deleted file mode 100644 index e9c04ff..0000000 --- a/internal/mirror/git.go +++ /dev/null @@ -1,49 +0,0 @@ -package mirror - -import ( - "context" - "errors" - - "github.com/git-pkgs/proxy/internal/gitcache" -) - -// GitMirrorer caches Git repositories for pkg:git items. location is the -// repository's host and path, such as codeberg.org/forgejo/forgejo, and -// version a full commit ID or a tag, or empty to cache the repository's -// branches and tags only. -type GitMirrorer interface { - Mirror(ctx context.Context, location, version string) (gitcache.MirrorResult, error) -} - -// UnavailableGit fails every pkg:git item with Reason. -type UnavailableGit struct { - Reason string -} - -func (u UnavailableGit) Mirror(context.Context, string, string) (gitcache.MirrorResult, error) { - return gitcache.MirrorResult{}, errors.New(u.Reason) -} - -var errNoGitUpstream = errors.New("git repositories are mirrored from upstream.git, and none is configured") - -// mirrorGit caches a pkg:git repository, and the commit its version names -// with the submodules it pins, in the cache /git serves clones from. -func (m *Mirror) mirrorGit(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - if m.Git == nil { - m.recordFailure(tracker, pv, errNoGitUpstream) - return - } - result, err := m.Git.Mirror(ctx, pv.Name, pv.Version) - if err != nil { - m.recordFailure(tracker, pv, err) - return - } - if result.Cached { - tracker.skipped.Add(1) - m.logger.Debug("already cached", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version) - return - } - tracker.completed.Add(1) - tracker.bytes.Add(result.Bytes) - m.logger.Info("mirrored", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "size", result.Bytes) -} diff --git a/internal/mirror/git_test.go b/internal/mirror/git_test.go deleted file mode 100644 index aa3aed4..0000000 --- a/internal/mirror/git_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package mirror - -import ( - "context" - "errors" - "sync" - "testing" - - "github.com/git-pkgs/proxy/internal/gitcache" -) - -type gitCall struct{ location, commit string } - -type fakeGit struct { - mu sync.Mutex - calls []gitCall - result gitcache.MirrorResult - err error -} - -func (f *fakeGit) Mirror(_ context.Context, location, commit string) (gitcache.MirrorResult, error) { - f.mu.Lock() - defer f.mu.Unlock() - f.calls = append(f.calls, gitCall{location, commit}) - return f.result, f.err -} - -const gitTestCommit = "a72d2c07cfca03b55371089de6aa230d8c951fa0" - -func TestMirrorRunCountsGitRepositories(t *testing.T) { - tests := []struct { - name string - result gitcache.MirrorResult - err error - wantCompleted, wantSkipped, wantFailed int64 - }{ - {name: "fetched", result: gitcache.MirrorResult{Bytes: 42}, wantCompleted: 1}, - {name: "cached", result: gitcache.MirrorResult{Cached: true}, wantSkipped: 1}, - {name: "failed", err: errors.New("boom"), wantFailed: 1}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - m := setupTestMirror(t, 1) - git := &fakeGit{result: tt.result, err: tt.err} - m.Git = git - progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo@" + gitTestCommit}}) - if err != nil { - t.Fatalf("Run: %v", err) - } - if progress.Completed != tt.wantCompleted || progress.Skipped != tt.wantSkipped || progress.Failed != tt.wantFailed { - t.Errorf("progress = %+v", progress) - } - if progress.Bytes != tt.result.Bytes { - t.Errorf("bytes = %d, want %d", progress.Bytes, tt.result.Bytes) - } - want := gitCall{"codeberg.org/forgejo/forgejo", gitTestCommit} - if len(git.calls) != 1 || git.calls[0] != want { - t.Errorf("calls = %+v, want %+v", git.calls, want) - } - }) - } -} - -// TestMirrorRunGitRepositoryWithoutCommit checks a pkg:git PURL without a -// version mirrors the repository instead of listing versions. -func TestMirrorRunGitRepositoryWithoutCommit(t *testing.T) { - m := setupTestMirror(t, 1) - git := &fakeGit{} - m.Git = git - versions := &fakeVersions{err: errors.New("listed versions")} - progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo"}, Versions: versions}) - if err != nil { - t.Fatalf("Run: %v", err) - } - if len(versions.asked) != 0 { - t.Errorf("listed versions of %v", versions.asked) - } - if progress.Failed != 0 || len(git.calls) != 1 || git.calls[0] != (gitCall{"codeberg.org/forgejo/forgejo", ""}) { - t.Errorf("progress = %+v, calls = %+v", progress, git.calls) - } -} - -func TestMirrorRunGitWithoutUpstream(t *testing.T) { - m := setupTestMirror(t, 1) - progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo@" + gitTestCommit}}) - if err != nil { - t.Fatalf("Run: %v", err) - } - if progress.Failed != 1 { - t.Errorf("progress = %+v, want one failure", progress) - } -} diff --git a/internal/mirror/helm.go b/internal/mirror/helm.go deleted file mode 100644 index bda16dd..0000000 --- a/internal/mirror/helm.go +++ /dev/null @@ -1,40 +0,0 @@ -package mirror - -import ( - "context" - "errors" - "strings" - - "github.com/git-pkgs/proxy/internal/handler" -) - -// ChartMirrorer caches Helm charts from the configured repositories the way -// the Helm handler serves them. -type ChartMirrorer interface { - MirrorChart(ctx context.Context, repository, chart, version string) (*handler.CacheResult, error) -} - -var ( - errHelmNoRepository = errors.New("helm charts need the upstream.helm repository as the namespace, such as pkg:helm/{repository}/{chart}@{version}") - errHelmUnavailable = errors.New("mirroring Helm charts isn't available here") -) - -// mirrorHelm mirrors a chart release. The PURL namespace names the -// upstream.helm repository, since the handler caches charts per repository. -func (m *Mirror) mirrorHelm(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - repository, chart, ok := strings.Cut(pv.Name, "/") - if !ok || repository == "" || chart == "" { - m.recordFailure(tracker, pv, errHelmNoRepository) - return - } - if m.Charts == nil { - m.recordFailure(tracker, pv, errHelmUnavailable) - return - } - result, err := m.Charts.MirrorChart(ctx, repository, chart, pv.Version) - if err != nil { - m.recordFailure(tracker, pv, err) - return - } - m.recordResult(tracker, pv, result) -} diff --git a/internal/mirror/image.go b/internal/mirror/image.go deleted file mode 100644 index 5116f26..0000000 --- a/internal/mirror/image.go +++ /dev/null @@ -1,165 +0,0 @@ -package mirror - -import ( - "context" - "errors" - "fmt" - "strings" - - "github.com/git-pkgs/proxy/internal/handler" -) - -// ImageMirrorer fetches container images into the cache. -type ImageMirrorer interface { - MirrorImage(ctx context.Context, ref handler.ImageRef, platforms []handler.Platform) (handler.ImageResult, error) -} - -// UnavailableImages refuses every image with the given reason. -type UnavailableImages struct { - Reason string -} - -func (u UnavailableImages) MirrorImage(context.Context, handler.ImageRef, []handler.Platform) (handler.ImageResult, error) { - return handler.ImageResult{}, errors.New(u.Reason) -} - -func isImageEcosystem(ecosystem string) bool { - return ecosystem == "docker" || ecosystem == "oci" -} - -// BottleResolver resolves a Homebrew formula version to the image its -// bottles are published under. -type BottleResolver interface { - BottleImage(ctx context.Context, formula, version string) (handler.ImageRef, error) -} - -// takesPlatforms reports whether --platform applies to an ecosystem: images, -// and Homebrew bottles, which are published as images. -func takesPlatforms(ecosystem string) bool { - return isImageEcosystem(ecosystem) || ecosystem == "brew" -} - -// ParsePlatforms parses os/architecture[/variant] values. -func ParsePlatforms(values []string) ([]handler.Platform, error) { - var platforms []handler.Platform - for _, value := range values { - for _, item := range strings.Split(value, ",") { - item = strings.TrimSpace(item) - if item == "" { - continue - } - platform, err := handler.ParsePlatform(item) - if err != nil { - return nil, err - } - platforms = append(platforms, platform) - } - } - return platforms, nil -} - -// imageRef maps a docker or oci PURL to the image reference and platforms to -// mirror. -// -// - pkg:docker//@, optionally with -// repository_url=[/]. -// - pkg:oci/@?repository_url=/, optionally -// with tag= and arch=. OCI PURLs have no default -// registry, so repository_url is required. -func imageRef(pv PackageVersion) (handler.ImageRef, []handler.Platform, error) { - platforms, err := ParsePlatforms(pv.Platforms) - if err != nil { - return handler.ImageRef{}, nil, err - } - ref := handler.ImageRef{Name: pv.Name} - if pv.Version != "" { - ref.References = []string{pv.Version} - } - repositoryURL := pv.Qualifiers["repository_url"] - if pv.Ecosystem == "docker" && repositoryURL == "" { - host, repository, found := strings.Cut(pv.Name, "/") - if found && (strings.ContainsAny(host, ".:") || host == "localhost") { - ref.Registry = host - ref.Name = repository - } - } - if pv.Ecosystem == "oci" { - if repositoryURL == "" { - return ref, nil, errors.New("pkg:oci needs a repository_url qualifier") - } - if tag := pv.Qualifiers["tag"]; tag != "" && tag != pv.Version { - ref.References = append(ref.References, tag) - } - if arch := pv.Qualifiers["arch"]; arch != "" && len(platforms) == 0 { - platforms = []handler.Platform{{Architecture: arch}} - } - } - if repositoryURL != "" { - repositoryURL = strings.TrimPrefix(strings.TrimPrefix(repositoryURL, "https://"), "http://") - host, repository, _ := strings.Cut(strings.Trim(repositoryURL, "/"), "/") - ref.Registry = host - switch { - case pv.Ecosystem == "oci" && repository != "": - // The repository URL names the repository itself. - ref.Name = repository - case repository != "": - ref.Name = repository + "/" + pv.Name - } - } - return ref, platforms, nil -} - -func (m *Mirror) mirrorImage(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - ref, platforms, err := imageRef(pv) - m.mirrorImageRef(ctx, pv, ref, platforms, err, tracker) -} - -// mirrorBrew mirrors the bottles of a Homebrew formula version, which brew -// pulls as an image from homebrew/core on its artifact domain. -func (m *Mirror) mirrorBrew(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - if m.Images == nil || m.Bottles == nil { - m.recordFailure(tracker, pv, errors.New("mirroring Homebrew bottles isn't available here")) - return - } - platforms, err := ParsePlatforms(pv.Platforms) - if err != nil { - m.recordFailure(tracker, pv, err) - return - } - ref, err := m.Bottles.BottleImage(ctx, pv.Name, pv.Version) - m.mirrorImageRef(ctx, pv, ref, platforms, err, tracker) -} - -// mirrorImageRef mirrors the image ref for pv, or records refErr, the error -// resolving it. -func (m *Mirror) mirrorImageRef(ctx context.Context, pv PackageVersion, ref handler.ImageRef, platforms []handler.Platform, refErr error, tracker *progressTracker) { - fail := func(err error) { - tracker.failed.Add(1) - tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) - m.logger.Warn("mirror failed", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) - } - if m.proxy != nil && m.proxy.Denylist.Denied(pv.String()) { - fail(fmt.Errorf("%w: %s", handler.ErrVersionDenied, pv)) - return - } - if refErr != nil { - fail(refErr) - return - } - result, err := m.Images.MirrorImage(ctx, ref, platforms) - for _, warning := range result.Warnings { - m.logger.Warn("mirror warning", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "warning", warning) - } - tracker.bytes.Add(result.Bytes) - if err != nil { - fail(err) - return - } - if result.Fetched == 0 { - tracker.skipped.Add(1) - m.logger.Debug("already cached", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version) - return - } - tracker.completed.Add(1) - m.logger.Info("mirrored", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "size", result.Bytes) -} diff --git a/internal/mirror/image_test.go b/internal/mirror/image_test.go deleted file mode 100644 index c38ddf7..0000000 --- a/internal/mirror/image_test.go +++ /dev/null @@ -1,190 +0,0 @@ -package mirror - -import ( - "context" - "errors" - "reflect" - "strings" - "sync" - "testing" - - "github.com/git-pkgs/proxy/internal/denylist" - "github.com/git-pkgs/proxy/internal/handler" -) - -func TestImageRefFromPURL(t *testing.T) { - const digest = "sha256:1111111111111111111111111111111111111111111111111111111111111111" - tests := []struct { - purl string - platforms []string - wantRef handler.ImageRef - wantPlatforms []handler.Platform - wantErr string - }{ - { - purl: "pkg:docker/library/alpine@3.20", - wantRef: handler.ImageRef{Name: "library/alpine", References: []string{"3.20"}}, - }, - { - purl: "pkg:docker/ghcr.io/team/app@1.0", - wantRef: handler.ImageRef{Registry: "ghcr.io", Name: "team/app", References: []string{"1.0"}}, - }, - { - purl: "pkg:docker/registry.example:5000/team/app@" + digest, - wantRef: handler.ImageRef{Registry: "registry.example:5000", Name: "team/app", References: []string{digest}}, - }, - { - purl: "pkg:docker/localhost/app@1.0", - wantRef: handler.ImageRef{Registry: "localhost", Name: "app", References: []string{"1.0"}}, - }, - { - purl: "pkg:docker/localhost:5000/team/app@1.0", - wantRef: handler.ImageRef{Registry: "localhost:5000", Name: "team/app", References: []string{"1.0"}}, - }, - { - purl: "pkg:docker/alpine@3.20?repository_url=ghcr.io", - platforms: []string{"linux/arm64/v8"}, - wantRef: handler.ImageRef{Registry: "ghcr.io", Name: "alpine", References: []string{"3.20"}}, - wantPlatforms: []handler.Platform{{OS: "linux", Architecture: "arm64", Variant: "v8"}}, - }, - { - purl: "pkg:docker/app@1.0?repository_url=https://registry.example/team", - wantRef: handler.ImageRef{Registry: "registry.example", Name: "team/app", References: []string{"1.0"}}, - }, - { - purl: "pkg:oci/debian@" + digest + "?repository_url=docker.io/library/debian&tag=bookworm&arch=arm64", - wantRef: handler.ImageRef{Registry: "docker.io", Name: "library/debian", References: []string{digest, "bookworm"}}, - wantPlatforms: []handler.Platform{{Architecture: "arm64"}}, - }, - { - purl: "pkg:oci/debian@" + digest + "?repository_url=docker.io/library/debian&arch=arm64", - platforms: []string{"linux/amd64"}, - wantRef: handler.ImageRef{Registry: "docker.io", Name: "library/debian", References: []string{digest}}, - wantPlatforms: []handler.Platform{{OS: "linux", Architecture: "amd64"}}, - }, - { - purl: "pkg:oci/debian@" + digest, - wantErr: "repository_url", - }, - } - for _, tt := range tests { - t.Run(tt.purl, func(t *testing.T) { - var items []PackageVersion - source := &PURLSource{PURLs: []string{tt.purl}, Platforms: tt.platforms} - if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { - items = append(items, pv) - return nil - }); err != nil { - t.Fatalf("Enumerate: %v", err) - } - if len(items) != 1 { - t.Fatalf("items = %d, want 1", len(items)) - } - ref, platforms, err := imageRef(items[0]) - if tt.wantErr != "" { - if err == nil || !strings.Contains(err.Error(), tt.wantErr) { - t.Fatalf("err = %v, want %q", err, tt.wantErr) - } - return - } - if err != nil { - t.Fatalf("imageRef: %v", err) - } - if !reflect.DeepEqual(ref, tt.wantRef) { - t.Errorf("ref = %+v, want %+v", ref, tt.wantRef) - } - if !reflect.DeepEqual(platforms, tt.wantPlatforms) { - t.Errorf("platforms = %+v, want %+v", platforms, tt.wantPlatforms) - } - }) - } -} - -func TestUnversionedImagePURLIsNotExpanded(t *testing.T) { - var items []PackageVersion - source := &PURLSource{PURLs: []string{"pkg:docker/library/alpine"}} - if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { - items = append(items, pv) - return nil - }); err != nil { - t.Fatalf("Enumerate: %v", err) - } - if len(items) != 1 || items[0].Version != "" { - t.Fatalf("items = %+v, want one unversioned item", items) - } -} - -type fakeImages struct { - mu sync.Mutex - refs []handler.ImageRef - result handler.ImageResult - err error -} - -func (f *fakeImages) MirrorImage(_ context.Context, ref handler.ImageRef, _ []handler.Platform) (handler.ImageResult, error) { - f.mu.Lock() - defer f.mu.Unlock() - f.refs = append(f.refs, ref) - return f.result, f.err -} - -func TestMirrorRunCountsImages(t *testing.T) { - tests := []struct { - name string - result handler.ImageResult - err error - wantCompleted int64 - wantSkipped int64 - wantFailed int64 - }{ - {name: "fetched", result: handler.ImageResult{Fetched: 3, Bytes: 42}, wantCompleted: 1}, - {name: "cached", result: handler.ImageResult{}, wantSkipped: 1}, - {name: "failed", err: errors.New("boom"), wantFailed: 1}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - m := setupTestMirror(t, 1) - images := &fakeImages{result: tt.result, err: tt.err} - m.Images = images - progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:docker/library/alpine@3.20"}}) - if err != nil { - t.Fatalf("Run: %v", err) - } - if progress.Completed != tt.wantCompleted || progress.Skipped != tt.wantSkipped || progress.Failed != tt.wantFailed { - t.Errorf("progress = %+v", progress) - } - if progress.Bytes != tt.result.Bytes { - t.Errorf("bytes = %d, want %d", progress.Bytes, tt.result.Bytes) - } - if len(images.refs) != 1 || images.refs[0].Name != "library/alpine" { - t.Errorf("refs = %+v", images.refs) - } - }) - } -} - -func TestMirrorRunDeniedImage(t *testing.T) { - m := setupTestMirror(t, 1) - policy, err := denylist.New([]string{"pkg:docker/library/alpine@3.20"}) - if err != nil { - t.Fatal(err) - } - m.proxy.Denylist = policy - images := &fakeImages{} - m.Images = images - - progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:docker/library/alpine@3.20"}}) - if err != nil { - t.Fatalf("Run: %v", err) - } - if progress.Failed != 1 || len(images.refs) != 0 { - t.Errorf("denied image: failed = %d, mirror calls = %d", progress.Failed, len(images.refs)) - } -} - -func TestJobRequestRejectsInvalidPlatform(t *testing.T) { - js := &JobStore{} - if _, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:docker/library/alpine@3.20"}, Platforms: []string{"linux"}}); err == nil { - t.Error("invalid platform was accepted") - } -} diff --git a/internal/mirror/job.go b/internal/mirror/job.go index 841c7c9..fbae2a2 100644 --- a/internal/mirror/job.go +++ b/internal/mirror/job.go @@ -3,7 +3,6 @@ package mirror import ( "context" "crypto/rand" - "encoding/json" "fmt" "sync" "time" @@ -36,14 +35,8 @@ type Job struct { // JobRequest is the JSON body for starting a mirror job via the API. type JobRequest struct { - PURLs []string `json:"purls,omitempty"` - SBOM json.RawMessage `json:"sbom,omitempty"` - Registry string `json:"registry,omitempty"` - // Platforms limit which platforms of container images are mirrored, - // as os/architecture[/variant]. - Platforms []string `json:"platforms,omitempty"` - // Latest mirrors only the newest version of PURLs without a version. - Latest bool `json:"latest,omitempty"` + PURLs []string `json:"purls,omitempty"` + Registry string `json:"registry,omitempty"` } // JobStore manages in-memory mirror jobs. @@ -52,8 +45,6 @@ type JobStore struct { jobs map[string]*Job mirror *Mirror parentCtx context.Context - // running counts the jobs whose goroutine has not returned yet. - running sync.WaitGroup } // NewJobStore creates a new job store. The parent context is used as the base @@ -87,7 +78,7 @@ func (js *JobStore) Create(req JobRequest) (string, error) { js.jobs[id] = job js.mu.Unlock() - js.running.Go(func() { js.runJob(ctx, cancel, job, source) }) + go js.runJob(ctx, cancel, job, source) return id, nil } @@ -129,12 +120,6 @@ func (js *JobStore) Cancel(id string) bool { return true } -// Wait blocks until every job has returned. Cancel the parent context first -// to stop running jobs. -func (js *JobStore) Wait() { - js.running.Wait() -} - // Cleanup removes completed/failed/canceled jobs older than jobTTL. func (js *JobStore) Cleanup() { js.mu.Lock() @@ -204,20 +189,13 @@ func (js *JobStore) runJob(ctx context.Context, cancel context.CancelFunc, job * } func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint:ireturn // interface return is the design - if _, err := ParsePlatforms(req.Platforms); err != nil { - return nil, err - } switch { - case len(req.PURLs) > 0 && len(req.SBOM) > 0: - return nil, fmt.Errorf("request must include only one of purls or sbom") case len(req.PURLs) > 0: - return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil - case len(req.SBOM) > 0: - return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil + return &PURLSource{PURLs: req.PURLs}, nil case req.Registry != "": return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead") default: - return nil, fmt.Errorf("request must include purls or sbom") + return nil, fmt.Errorf("request must include purls") } } diff --git a/internal/mirror/job_test.go b/internal/mirror/job_test.go index 7b4c88e..f7f2f1c 100644 --- a/internal/mirror/job_test.go +++ b/internal/mirror/job_test.go @@ -2,27 +2,13 @@ package mirror import ( "context" - "encoding/json" "testing" "time" ) -// newTestJobStore returns a job store whose jobs are stopped and waited for -// before the test's database and directories are removed. -func newTestJobStore(t *testing.T) *JobStore { - t.Helper() - m := setupTestMirror(t, 1) - ctx, cancel := context.WithCancel(context.Background()) - js := NewJobStore(ctx, m) - t.Cleanup(func() { - cancel() - js.Wait() - }) - return js -} - func TestJobStoreCreateAndGet(t *testing.T) { - js := newTestJobStore(t) + m := setupTestMirror(t, 1) + js := NewJobStore(context.Background(), m) id, err := js.Create(JobRequest{ PURLs: []string{"pkg:npm/lodash@4.17.21"}, @@ -48,7 +34,8 @@ func TestJobStoreCreateAndGet(t *testing.T) { } func TestJobStoreGetNotFound(t *testing.T) { - js := newTestJobStore(t) + m := setupTestMirror(t, 1) + js := NewJobStore(context.Background(), m) job := js.Get("nonexistent") if job != nil { @@ -57,7 +44,8 @@ func TestJobStoreGetNotFound(t *testing.T) { } func TestJobStoreCancelNotFound(t *testing.T) { - js := newTestJobStore(t) + m := setupTestMirror(t, 1) + js := NewJobStore(context.Background(), m) if js.Cancel("nonexistent") { t.Error("expected Cancel to return false for nonexistent job") @@ -65,7 +53,8 @@ func TestJobStoreCancelNotFound(t *testing.T) { } func TestJobStoreCreateInvalidRequest(t *testing.T) { - js := newTestJobStore(t) + m := setupTestMirror(t, 1) + js := NewJobStore(context.Background(), m) _, err := js.Create(JobRequest{}) if err == nil { @@ -74,7 +63,8 @@ func TestJobStoreCreateInvalidRequest(t *testing.T) { } func TestJobStoreMultipleJobs(t *testing.T) { - js := newTestJobStore(t) + m := setupTestMirror(t, 1) + js := NewJobStore(context.Background(), m) id1, err := js.Create(JobRequest{PURLs: []string{"pkg:npm/lodash@4.17.21"}}) if err != nil { @@ -98,7 +88,8 @@ func TestJobStoreMultipleJobs(t *testing.T) { } func TestSourceFromRequestPURLs(t *testing.T) { - js := newTestJobStore(t) + m := setupTestMirror(t, 1) + js := NewJobStore(context.Background(), m) source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/lodash@1.0.0"}}) if err != nil { @@ -109,37 +100,9 @@ func TestSourceFromRequestPURLs(t *testing.T) { } } -func TestSourceFromRequestSBOM(t *testing.T) { - js := newTestJobStore(t) - sbom := json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`) - - source, err := js.sourceFromRequest(JobRequest{SBOM: sbom}) - if err != nil { - t.Fatalf("sourceFromRequest() error = %v", err) - } - sbomSource, ok := source.(*SBOMSource) - if !ok { - t.Fatalf("expected *SBOMSource, got %T", source) - } - if got := string(sbomSource.Data); got != string(sbom) { - t.Errorf("SBOM data = %q, want %q", got, sbom) - } -} - -func TestSourceFromRequestRejectsPURLsAndSBOM(t *testing.T) { - js := newTestJobStore(t) - - _, err := js.sourceFromRequest(JobRequest{ - PURLs: []string{"pkg:npm/lodash@4.17.21"}, - SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`), - }) - if err == nil { - t.Fatal("expected error when both purls and sbom are provided") - } -} - func TestSourceFromRequestRegistryRejected(t *testing.T) { - js := newTestJobStore(t) + m := setupTestMirror(t, 1) + js := NewJobStore(context.Background(), m) _, err := js.sourceFromRequest(JobRequest{Registry: "npm"}) if err == nil { @@ -148,7 +111,8 @@ func TestSourceFromRequestRegistryRejected(t *testing.T) { } func TestJobStoreCleanup(t *testing.T) { - js := newTestJobStore(t) + m := setupTestMirror(t, 1) + js := NewJobStore(context.Background(), m) // Add a completed job with old CreatedAt js.mu.Lock() @@ -183,7 +147,8 @@ func TestJobStoreCleanup(t *testing.T) { } func TestJobStoreCancelPreservesStateAfterRunJob(t *testing.T) { - js := newTestJobStore(t) + m := setupTestMirror(t, 1) + js := NewJobStore(context.Background(), m) // Create a job with a PURL that will fail (no real upstream in test) id, err := js.Create(JobRequest{PURLs: []string{"pkg:npm/nonexistent-pkg@0.0.0"}}) diff --git a/internal/mirror/julia.go b/internal/mirror/julia.go deleted file mode 100644 index 613ba76..0000000 --- a/internal/mirror/julia.go +++ /dev/null @@ -1,84 +0,0 @@ -package mirror - -import ( - "context" - "errors" - "fmt" - "strings" - - "github.com/git-pkgs/proxy/internal/handler" - "github.com/git-pkgs/registries" -) - -// PackageFinder finds a package's registry entry, which for Julia gives the -// UUID the package is registered under. -type PackageFinder interface { - FindPackage(ctx context.Context, ecosystem, name string) (*registries.Package, error) -} - -// FindPackage implements PackageFinder. -func (r RegistryVersions) FindPackage(ctx context.Context, ecosystem, name string) (*registries.Package, error) { - reg, err := r.registry(ecosystem, name) - if err != nil { - return nil, err - } - return reg.FetchPackage(ctx, name) -} - -// mirrorJulia fetches the tarball unchecked because Pkg checks the tree hash -// of what it extracts. -func (m *Mirror) mirrorJulia(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - fail := func(err error) { m.recordFailure(tracker, pv, err) } - uuid := strings.ToLower(pv.Qualifiers["uuid"]) - if uuid == "" { - fail(fmt.Errorf("julia packages need a uuid qualifier, such as pkg:julia/%s@%s?uuid=", pv.Name, pv.Version)) - return - } - - pkg, err := m.packageFinder().FindPackage(ctx, pv.Ecosystem, pv.Name) - if err != nil { - fail(err) - return - } - registered, _ := pkg.Metadata["uuid"].(string) - if !strings.EqualFold(registered, uuid) { - fail(fmt.Errorf("%s in the General registry has uuid %s, not %s; only General packages can be mirrored", pv.Name, registered, uuid)) - return - } - // The handler names packages as the registry does. - name := pkg.Name - if name == "" { - name = pv.Name - } - - v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, name, pv.Version) - if err != nil { - fail(err) - return - } - treeHash, _ := v.Metadata["git-tree-sha1"].(string) - if treeHash == "" { - fail(errors.New("the registry lists no tree hash for this version")) - return - } - url, filename, err := handler.JuliaPackageDownload(m.JuliaUpstream, uuid, treeHash) - if err != nil { - fail(err) - return - } - result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, pv.Ecosystem, name, treeHash, filename, url) - if err != nil { - fail(err) - return - } - m.recordResult(tracker, pv, result) -} - -// packageFinder finds packages where Versions lists versions, so both come -// from the same registry. -func (m *Mirror) packageFinder() PackageFinder { //nolint:ireturn // any finder will do - if finder, ok := m.Versions.(PackageFinder); ok { - return finder - } - return RegistryVersions{} -} diff --git a/internal/mirror/latest_test.go b/internal/mirror/latest_test.go deleted file mode 100644 index 8135116..0000000 --- a/internal/mirror/latest_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package mirror - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "reflect" - "testing" -) - -func TestPURLSourceLatestMirrorsNewestVersion(t *testing.T) { - versions := &fakeVersions{versions: []string{"1.0.0", "1.10.0", "1.2.0"}} - source := &PURLSource{PURLs: []string{"pkg:npm/left-pad", "pkg:npm/is-odd@3.0.0"}, Versions: versions, Latest: true} - - items := enumerate(t, source) - - want := []PackageVersion{ - {Ecosystem: "npm", Name: "left-pad", Version: "1.10.0"}, - {Ecosystem: "npm", Name: "is-odd", Version: "3.0.0"}, - } - if !reflect.DeepEqual(items, want) { - t.Errorf("items = %v, want %v", items, want) - } -} - -func TestPURLSourceLatestWithoutVersions(t *testing.T) { - source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{}, Latest: true} - if items := enumerate(t, source); len(items) != 0 { - t.Errorf("items = %v, want none", items) - } -} - -func TestSBOMSourceLatest(t *testing.T) { - bom := map[string]any{ - "bomFormat": "CycloneDX", - "specVersion": "1.4", - "components": []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}}, - } - source := &SBOMSource{Data: marshalJSON(t, bom), Versions: &fakeVersions{versions: []string{"1.0.0", "1.3.0"}}, Latest: true} - - items := enumerate(t, source) - - if len(items) != 1 || items[0].Version != "1.3.0" { - t.Errorf("items = %v, want only left-pad@1.3.0", items) - } -} - -// TestRegistryVersionsLatestSkipsYanked checks that without an advertised -// latest version the newest version by publish time that isn't yanked wins. -func TestRegistryVersionsLatestSkipsYanked(t *testing.T) { - registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/v1/crates/demo" { - http.NotFound(w, r) - return - } - _ = json.NewEncoder(w).Encode(map[string]any{ - "crate": map[string]any{"name": "demo"}, - "versions": []map[string]any{ - {"num": "1.3.0", "yanked": true, "created_at": "2026-03-01T00:00:00Z"}, - {"num": "1.2.0", "created_at": "2026-02-01T00:00:00Z"}, - {"num": "1.10.0-alpha.1", "created_at": "2026-01-01T00:00:00Z"}, - }, - }) - })) - defer registry.Close() - versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }} - - got, err := versions.LatestVersion(context.Background(), "cargo", "demo") - if err != nil { - t.Fatalf("LatestVersion() error = %v", err) - } - if got != "1.2.0" { - t.Errorf("LatestVersion() = %q, want 1.2.0", got) - } -} - -// TestRegistryVersionsLatestPrefersAdvertised checks that the version the -// registry advertises as latest wins over a higher one, here a 4.0.0 -// published by mistake and deprecated. -func TestRegistryVersionsLatestPrefersAdvertised(t *testing.T) { - registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/left-pad" { - http.NotFound(w, r) - return - } - _ = json.NewEncoder(w).Encode(map[string]any{ - "name": "left-pad", - "dist-tags": map[string]any{"latest": "3.4.0"}, - "versions": map[string]any{ - "3.4.0": map[string]any{"name": "left-pad", "version": "3.4.0"}, - "4.0.0": map[string]any{"name": "left-pad", "version": "4.0.0", "deprecated": "published by mistake"}, - }, - "time": map[string]any{"3.4.0": "2026-01-01T00:00:00Z", "4.0.0": "2026-02-01T00:00:00Z"}, - }) - })) - defer registry.Close() - versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }} - - got, err := versions.LatestVersion(context.Background(), "npm", "left-pad") - if err != nil { - t.Fatalf("LatestVersion() error = %v", err) - } - if got != "3.4.0" { - t.Errorf("LatestVersion() = %q, want the advertised 3.4.0", got) - } -} - -func TestSourceFromRequestPassesLatest(t *testing.T) { - js := NewJobStore(context.Background(), setupTestMirror(t, 1)) - - source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}, Latest: true}) - if err != nil { - t.Fatalf("sourceFromRequest() error = %v", err) - } - if !source.(*PURLSource).Latest { - t.Error("PURL source mirrors every version, want only the newest") - } - - source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`), Latest: true}) - if err != nil { - t.Fatalf("sourceFromRequest() error = %v", err) - } - if !source.(*SBOMSource).Latest { - t.Error("SBOM source mirrors every version, want only the newest") - } -} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index c08b1fb..26de7b8 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -3,10 +3,8 @@ package mirror import ( "context" - "errors" "fmt" "log/slog" - "strings" "sync" "sync/atomic" "time" @@ -24,41 +22,6 @@ type Mirror struct { storage storage.Storage logger *slog.Logger workers int - - // Images mirrors docker and oci items. Without it they fail like any - // other unsupported ecosystem. - Images ImageMirrorer - - // Charts mirrors helm charts. Without it they fail. - Charts ChartMirrorer - - // Git mirrors pkg:git repositories into the upstream.git cache. Without - // it they fail. - Git GitMirrorer - - // Bottles resolves Homebrew formula versions to the images their - // bottles are published under, which Images mirrors. Without it brew - // PURLs fail. - Bottles BottleResolver - - // Versions lists the versions of unversioned PURLs in mirror jobs and - // schedules. Nil lists them from the public registries. - Versions VersionLister - - // JuliaUpstream is the Pkg server Julia packages are fetched from. Empty - // uses https://pkg.julialang.org. - JuliaUpstream string - - // APKRepositories are the upstream.apk repositories by name. Nil uses - // the official Alpine mirror as "alpine". - APKRepositories map[string]string - - // RefusePackages, when set, fails every item Images does not take with - // this reason instead of fetching it, so nothing is cached. - RefusePackages string - - cran cranIndex - apk apkIndexes } // New creates a new Mirror with the given dependencies. @@ -240,75 +203,16 @@ func (m *Mirror) RunDryRun(ctx context.Context, source Source) ([]PackageVersion } func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - if isImageEcosystem(pv.Ecosystem) && m.Images != nil { - m.mirrorImage(ctx, pv, tracker) - return - } - if pv.Ecosystem == "brew" { - m.mirrorBrew(ctx, pv, tracker) - return - } - if m.RefusePackages != "" { - tracker.failed.Add(1) - tracker.addError(pv.Ecosystem, pv.Name, pv.Version, m.RefusePackages) - m.logger.Warn("mirror refused", - "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages) - return - } - if eco, ok := fileEcosystems[pv.Ecosystem]; ok { - m.mirrorFiles(ctx, pv, eco, tracker) - return - } - if pv.Ecosystem == "composer" { - m.mirrorComposer(ctx, pv, tracker) - return - } - if pv.Ecosystem == "helm" { - m.mirrorHelm(ctx, pv, tracker) - return - } - if pv.Ecosystem == "julia" { - m.mirrorJulia(ctx, pv, tracker) - return - } - if pv.Ecosystem == "swift" { - m.mirrorSwift(ctx, pv, tracker) - return - } - if pv.Ecosystem == "cran" { - m.mirrorCRAN(ctx, pv, tracker) - return - } - if pv.Ecosystem == "alpine" { - m.mirrorAPK(ctx, pv, tracker) - return - } - if pv.Ecosystem == "git" { - m.mirrorGit(ctx, pv, tracker) - return - } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { - m.recordFailure(tracker, pv, err) + tracker.failed.Add(1) + tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) + m.logger.Warn("mirror failed", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) return } - m.recordResult(tracker, pv, result) -} -// recordFailure counts and logs a package that could not be mirrored. -func (m *Mirror) recordFailure(tracker *progressTracker, pv PackageVersion, err error) { - tracker.failed.Add(1) - tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) - m.logger.Warn("mirror failed", - "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) -} - -// recordResult closes the fetched artifact and counts it as mirrored, or as -// skipped when it was already cached. -func (m *Mirror) recordResult(tracker *progressTracker, pv PackageVersion, result *handler.CacheResult) { - if result.Reader != nil { - _ = result.Reader.Close() - } + _ = result.Reader.Close() if result.Cached { tracker.skipped.Add(1) @@ -316,119 +220,9 @@ func (m *Mirror) recordResult(tracker *progressTracker, pv PackageVersion, resul "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version) } else { tracker.completed.Add(1) - tracker.bytes.Add(result.Artifact.Size) + tracker.bytes.Add(result.Size) m.logger.Info("mirrored", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, - "size", result.Artifact.Size) + "size", result.Size) } } - -// mirrorComposer mirrors the archive of a tagged Composer release under the -// file name the Composer handler serves it as, checked against the listed -// shasum, and records the release's publish time, which the handler needs to -// serve the archive from the cache without the metadata. -func (m *Mirror) mirrorComposer(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - // Packagist names are lowercase, and so are the paths Composer requests. - name := strings.ToLower(pv.Name) - v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, name, pv.Version) - if err != nil { - m.recordFailure(tracker, pv, err) - return - } - distURL, _ := v.Metadata["dist_url"].(string) - distType, _ := v.Metadata["dist_type"].(string) - filename, declared, err := handler.ComposerReleaseArchive(v.Number, distURL, distType, strings.TrimPrefix(v.Integrity, "sha1-")) - if err != nil { - m.recordFailure(tracker, pv, err) - return - } - result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, name, v.Number, filename, distURL, declared) - if err != nil { - m.recordFailure(tracker, pv, err) - return - } - if err := m.proxy.RecordComposerPublishTime(name, v.Number, v.PublishedAt); err != nil { - m.logger.Warn("failed to store composer publish time", - "name", name, "version", v.Number, "error", err) - } - m.recordResult(tracker, pv, result) -} - -// mirrorFiles mirrors every file of a version, or those its qualifiers -// select. Each file is cached under the name and version the download -// handler reads from its filename, so clients are served it. -func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, eco fileEcosystem, tracker *progressTracker) { - fail := func(err error) { m.recordFailure(tracker, pv, err) } - if eco.check != nil { - if err := eco.check(pv); err != nil { - fail(err) - return - } - } - v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, pv.Name, pv.Version) - if err != nil { - fail(fmt.Errorf("listing files: %w", err)) - return - } - var matched, fetched int - var size int64 - for _, file := range v.Artifacts { - if !eco.selects(pv, file) { - continue - } - name, version, filename, ok := eco.cacheKey(pv, file) - if !ok { - continue - } - matched++ - result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, filename, file.URL, fileDigest(file.Integrity)) - if err != nil { - fail(fmt.Errorf("%s: %w", file.Filename, err)) - return - } - if result.Reader != nil { - _ = result.Reader.Close() - } - if !result.Cached { - fetched++ - size += result.Artifact.Size - } - } - switch selection := eco.selection(pv); { - case matched == 0 && selection != "": - fail(fmt.Errorf("no file matches %s", selection)) - case matched == 0: - fail(errors.New("version has no files")) - case fetched == 0: - tracker.skipped.Add(1) - m.logger.Debug("already cached", - "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version) - default: - tracker.completed.Add(1) - tracker.bytes.Add(size) - m.logger.Info("mirrored", - "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, - "files", fetched, "size", size) - } -} - -// fileDigest returns the SHA-256 of one file as a digest, which is checked -// against that file alone. The integrity GetOrFetchArtifactFromURLWithIntegrity -// takes is kept with the version instead, and the files of a version that -// share a name and version would overwrite each other's. Other checksums are -// not checked, as the download handlers don't check them either. -func fileDigest(integrity string) string { - if sum, ok := strings.CutPrefix(integrity, "sha256-"); ok { - return "sha256:" + sum - } - return "" -} - -// versionFinder finds versions where Versions lists them, so both come from -// the configured upstream. -func (m *Mirror) versionFinder() VersionFinder { //nolint:ireturn // any finder will do - if finder, ok := m.Versions.(VersionFinder); ok { - return finder - } - return RegistryVersions{} -} diff --git a/internal/mirror/mirror_test.go b/internal/mirror/mirror_test.go index 6852687..1d7d30d 100644 --- a/internal/mirror/mirror_test.go +++ b/internal/mirror/mirror_test.go @@ -2,13 +2,8 @@ package mirror import ( "context" - "database/sql" - "errors" "log/slog" - "net/http" "os" - "strings" - "sync" "testing" "time" @@ -48,14 +43,6 @@ func setupTestMirror(t *testing.T, workers int) *Mirror { const testPackageLodash = "lodash" -type signedURLStorage struct { - storage.Storage -} - -func (signedURLStorage) SignedURL(context.Context, string, time.Duration) (string, error) { - return "https://storage.example/artifact", nil -} - func TestMirrorRunEmptySource(t *testing.T) { m := setupTestMirror(t, 2) @@ -124,63 +111,6 @@ func TestMirrorRunCanceled(t *testing.T) { } } -func TestMirrorOneDirectServeCacheHit(t *testing.T) { - m := setupTestMirror(t, 1) - fetcher := fetch.NewFetcher( - fetch.WithHTTPClient(&http.Client{Transport: rejectingMirrorTransport{}}), fetch.WithMaxRetries(0)) - t.Cleanup(func() { _ = fetcher.Close() }) - m.proxy.Fetcher = fetcher - m.proxy.DirectServe = true - m.proxy.Storage = signedURLStorage{Storage: m.storage} - - packagePURL := "pkg:npm/example" - versionPURL := packagePURL + "@1.0.0" - if err := m.db.UpsertPackage(&database.Package{ - PURL: packagePURL, - Ecosystem: "npm", - Name: "example", - }); err != nil { - t.Fatalf("UpsertPackage() error = %v", err) - } - if err := m.db.UpsertVersion(&database.Version{ - PURL: versionPURL, - PackagePURL: packagePURL, - }); err != nil { - t.Fatalf("UpsertVersion() error = %v", err) - } - const storagePath = "npm/example/1.0.0/example-1.0.0.tgz" - size, hash, err := m.storage.Store(context.Background(), storagePath, strings.NewReader("tarball bytes")) - if err != nil { - t.Fatal(err) - } - if err := m.db.UpsertArtifact(&database.Artifact{ - VersionPURL: versionPURL, - Filename: "example-1.0.0.tgz", - UpstreamURL: "https://registry.example/artifact", - StoragePath: sql.NullString{String: storagePath, Valid: true}, - ContentHash: sql.NullString{String: hash, Valid: true}, - Size: sql.NullInt64{Int64: size, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - }); err != nil { - t.Fatalf("UpsertArtifact() error = %v", err) - } - - tracker := newProgressTracker() - m.mirrorOne(context.Background(), PackageVersion{ - Ecosystem: "npm", - Name: "example", - Version: "1.0.0", - }, tracker) - - progress := tracker.snapshot() - if progress.Skipped != 1 { - t.Errorf("skipped = %d, want 1", progress.Skipped) - } - if progress.Failed != 0 { - t.Errorf("failed = %d, want 0", progress.Failed) - } -} - func TestProgressTrackerSnapshot(t *testing.T) { pt := newProgressTracker() pt.total.Store(10) @@ -263,90 +193,3 @@ func TestNewMirrorDefaultWorkers(t *testing.T) { t.Errorf("workers = %d, want 1 (minimum)", m.workers) } } - -// recordingFetcher fails every fetch and records the URLs it was asked for, so -// a test can show nothing reached upstream. -type recordingFetcher struct { - mu sync.Mutex - urls []string -} - -func (f *recordingFetcher) record(url string) error { - f.mu.Lock() - defer f.mu.Unlock() - f.urls = append(f.urls, url) - return errors.New("recordingFetcher: no upstream in this test") -} - -func (f *recordingFetcher) Fetch(_ context.Context, url string) (*fetch.Artifact, error) { - return nil, f.record(url) -} - -func (f *recordingFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) { - return nil, f.record(url) -} - -func (f *recordingFetcher) Head(_ context.Context, url string) (int64, string, error) { - return 0, "", f.record(url) -} - -func (f *recordingFetcher) fetched() []string { - f.mu.Lock() - defer f.mu.Unlock() - return append([]string(nil), f.urls...) -} - -func TestMirrorRunRefusesPackages(t *testing.T) { - m := setupTestMirror(t, 2) - fetcher := &recordingFetcher{} - m.proxy.Fetcher = fetcher - const reason = "packages cannot be mirrored here" - m.RefusePackages = reason - - source := &PURLSource{PURLs: []string{"pkg:npm/lodash@4.17.21", "pkg:cargo/serde@1.0.0"}} - progress, err := m.Run(context.Background(), source) - if err != nil { - t.Fatalf("Run: %v", err) - } - - if progress.Failed != 2 || progress.Completed != 0 || progress.Skipped != 0 { - t.Errorf("progress = %d failed, %d completed, %d skipped; want 2 failed only", - progress.Failed, progress.Completed, progress.Skipped) - } - if len(progress.Errors) != 2 { - t.Fatalf("errors = %+v, want one per package", progress.Errors) - } - for _, e := range progress.Errors { - if e.Error != reason { - t.Errorf("%s/%s error = %q, want %q", e.Ecosystem, e.Name, e.Error, reason) - } - } - if urls := fetcher.fetched(); len(urls) != 0 { - t.Errorf("refused packages were fetched from upstream: %v", urls) - } - - stats, err := m.db.GetCacheStats() - if err != nil { - t.Fatalf("GetCacheStats: %v", err) - } - if stats.TotalArtifacts != 0 { - t.Errorf("cached artifacts = %d, want 0", stats.TotalArtifacts) - } -} - -// Images keep their own refusal, so a mirror refusing packages still hands -// docker and oci items to Images. -func TestMirrorRefusePackagesLeavesImagesToImages(t *testing.T) { - m := setupTestMirror(t, 1) - m.proxy.Fetcher = &recordingFetcher{} - m.RefusePackages = "packages refused" - m.Images = UnavailableImages{Reason: "images refused"} - - progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:docker/library/alpine@3.20"}}) - if err != nil { - t.Fatalf("Run: %v", err) - } - if len(progress.Errors) != 1 || progress.Errors[0].Error != "images refused" { - t.Errorf("errors = %+v, want the image refusal", progress.Errors) - } -} diff --git a/internal/mirror/registry.go b/internal/mirror/registry.go new file mode 100644 index 0000000..6b2c449 --- /dev/null +++ b/internal/mirror/registry.go @@ -0,0 +1,16 @@ +package mirror + +import ( + "context" + "fmt" +) + +// RegistrySource enumerates all packages in a registry for full mirroring. +// Registry enumeration is not yet implemented for any ecosystem. +type RegistrySource struct { + Ecosystem string +} + +func (s *RegistrySource) Enumerate(_ context.Context, _ func(PackageVersion) error) error { + return fmt.Errorf("registry enumeration is not yet implemented for ecosystem %q", s.Ecosystem) +} diff --git a/internal/mirror/registry_test.go b/internal/mirror/registry_test.go new file mode 100644 index 0000000..363bfea --- /dev/null +++ b/internal/mirror/registry_test.go @@ -0,0 +1,46 @@ +package mirror + +import ( + "context" + "testing" +) + +func TestRegistrySourceUnsupported(t *testing.T) { + source := &RegistrySource{Ecosystem: "golang"} + err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + return nil + }) + if err == nil { + t.Fatal("expected error for unsupported ecosystem") + } +} + +func TestRegistrySourceNPMNotImplemented(t *testing.T) { + source := &RegistrySource{Ecosystem: "npm"} + err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + return nil + }) + if err == nil { + t.Fatal("expected not-implemented error") + } +} + +func TestRegistrySourcePyPINotImplemented(t *testing.T) { + source := &RegistrySource{Ecosystem: "pypi"} + err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + return nil + }) + if err == nil { + t.Fatal("expected not-implemented error") + } +} + +func TestRegistrySourceCargoNotImplemented(t *testing.T) { + source := &RegistrySource{Ecosystem: "cargo"} + err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + return nil + }) + if err == nil { + t.Fatal("expected not-implemented error") + } +} diff --git a/internal/mirror/sbom_version_test.go b/internal/mirror/sbom_version_test.go deleted file mode 100644 index 43dde04..0000000 --- a/internal/mirror/sbom_version_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package mirror - -import ( - "context" - "net/http" - "reflect" - "testing" - - "github.com/git-pkgs/proxy/internal/handler" - "github.com/git-pkgs/registries" -) - -func TestSBOMSourceUsesComponentVersion(t *testing.T) { - for name, data := range map[string]string{ - "CycloneDX JSON": `{"components":[{"name":"app","version":"1.0","purl":"pkg:docker/ghcr.io/team/app"}]}`, - "CycloneDX XML": `app1.0pkg:docker/ghcr.io/team/app`, - "SPDX JSON": `{"spdxVersion":"SPDX-2.3","packages":[{"name":"app","versionInfo":"1.0","externalRefs":[{"referenceType":"purl","referenceLocator":"pkg:docker/ghcr.io/team/app"}]}]}`, - "SPDX tag-value": "DataLicense: CC0-1.0\nSPDXVersion: SPDX-2.3\nSPDXID: SPDXRef-DOCUMENT\nDocumentName: example\nDocumentNamespace: https://example.com/sbom\nPackageName: app\nSPDXID: SPDXRef-app\nPackageVersion: 1.0\nExternalRef: PACKAGE-MANAGER purl pkg:docker/ghcr.io/team/app\n", - } { - t.Run(name, func(t *testing.T) { - source := &SBOMSource{Data: []byte(data)} - var got []PackageVersion - if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { got = append(got, pv); return nil }); err != nil { - t.Fatal(err) - } - if len(got) != 1 || got[0].Version != "1.0" { - t.Fatalf("Enumerate = %+v, want one image at 1.0", got) - } - }) - } -} - -func TestSBOMSourceVersionPrecedence(t *testing.T) { - for _, test := range []struct{ name, componentVersion, imagePURL, wantVersion string }{ - {"PURL version wins", "2.0", "pkg:docker/team/app@1.0", "1.0"}, - {"digest from component", "sha256:abc123", "pkg:docker/team/app", "sha256:abc123"}, - {"unknown version", "", "pkg:docker/team/app", ""}, - {"package version", "1.0", "pkg:npm/example", "1.0"}, - } { - t.Run(test.name, func(t *testing.T) { - data := marshalJSON(t, map[string]any{"components": []map[string]any{{"name": "app", "version": test.componentVersion, "purl": test.imagePURL}}}) - var got []PackageVersion - source := &SBOMSource{Data: data, Versions: RegistryVersions{Client: registries.NewClient(registries.WithHTTPClient(&http.Client{Transport: rejectingMirrorTransport{}}))}} - if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { got = append(got, pv); return nil }); err != nil { - t.Fatal(err) - } - if len(got) != 1 || got[0].Version != test.wantVersion { - t.Fatalf("Enumerate = %+v, want version %q", got, test.wantVersion) - } - }) - } -} - -func TestMirrorRunSBOMDockerReference(t *testing.T) { - m := setupTestMirror(t, 1) - images := &fakeImages{result: handler.ImageResult{Fetched: 1}} - m.Images = images - source := &SBOMSource{ - Data: []byte(`{"components":[{"name":"app","version":"1.0","purl":"pkg:docker/registry.example:5000/team/app?arch=arm64"}]}`), - Platforms: []string{"linux/arm64"}, - } - progress, err := m.Run(context.Background(), source) - if err != nil { - t.Fatal(err) - } - if progress.Completed != 1 || progress.Failed != 0 { - t.Fatalf("progress = %+v", progress) - } - want := []handler.ImageRef{{Registry: "registry.example:5000", Name: "team/app", References: []string{"1.0"}}} - if !reflect.DeepEqual(images.refs, want) { - t.Errorf("image refs = %+v, want %+v", images.refs, want) - } -} diff --git a/internal/mirror/schedule.go b/internal/mirror/schedule.go deleted file mode 100644 index 1df9a75..0000000 --- a/internal/mirror/schedule.go +++ /dev/null @@ -1,151 +0,0 @@ -package mirror - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "log/slog" - "math/rand/v2" - "sync" - "time" -) - -const ( - maxFirstScheduleDelay = 10 * time.Minute - // scheduleJitter spreads later runs by ±interval/scheduleJitter. - scheduleJitter = 10 -) - -// Schedule mirrors a set of image PURLs every Interval. -type Schedule struct { - PURLs []string - Platforms []string - Interval time.Duration -} - -// Scheduler runs schedules in the background until its context ends. -type Scheduler struct { - mirror *Mirror - schedules []Schedule - logger *slog.Logger - wg sync.WaitGroup - // Claim, when set, reports whether this instance should run a schedule - // that no instance sharing the database started within minGap. Each - // instance keeps its own timer, so without it every instance would run - // every schedule. - Claim func(key string, minGap time.Duration) (bool, error) - // Release, when set, drops the claim of a run that was interrupted, so - // the next attempt runs it rather than waiting an interval. - Release func(key string) error - // randN returns a value in [0, n); tests replace it. - randN func(n int64) int64 -} - -// NewScheduler creates a scheduler that runs the schedules through m. -func NewScheduler(m *Mirror, schedules []Schedule, logger *slog.Logger) *Scheduler { - return &Scheduler{ - mirror: m, - schedules: schedules, - logger: logger, - randN: rand.Int64N, //nolint:gosec // jitter, not security - } -} - -// Start starts one goroutine per schedule. -func (s *Scheduler) Start(ctx context.Context) { - for i, schedule := range s.schedules { - if schedule.Interval <= 0 { - s.logger.Error("mirror schedule has no interval, skipped", "schedule", i) - continue - } - s.wg.Add(1) - go s.loop(ctx, i, schedule) - } -} - -// Wait blocks until every schedule has stopped after its context ended. -func (s *Scheduler) Wait() { - s.wg.Wait() -} - -func (s *Scheduler) loop(ctx context.Context, index int, schedule Schedule) { - defer s.wg.Done() - timer := time.NewTimer(scheduleDelay(schedule.Interval, true, s.randN)) - defer timer.Stop() - for { - select { - case <-ctx.Done(): - return - case <-timer.C: - } - s.run(ctx, index, schedule) - // The next run is timed from the end of this one, so runs never overlap. - timer.Reset(scheduleDelay(schedule.Interval, false, s.randN)) - } -} - -func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) { - key := scheduleKey(schedule) - if s.Claim != nil { - // Later runs come at least interval - 2*spread apart, so a run - // started more recently than that was another instance's. - minGap := schedule.Interval - 2*(schedule.Interval/scheduleJitter) //nolint:mnd // two spreads of margin - claimed, err := s.Claim(key, minGap) - if err != nil { - s.logger.Warn("mirror schedule: failed to claim run, skipping it", "schedule", index, "error", err) - return - } - if !claimed { - s.logger.Debug("mirror schedule: another instance ran it, skipping", "schedule", index) - return - } - } - started := time.Now() - progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms}) - if ctx.Err() != nil { - if s.Claim != nil && s.Release != nil { - if err := s.Release(key); err != nil { - s.logger.Warn("mirror schedule: failed to release interrupted run", "schedule", index, "error", err) - } - } - return - } - if err != nil { - s.logger.Error("mirror schedule failed", "schedule", index, "error", err) - return - } - level := slog.LevelInfo - if progress.Failed > 0 { - level = slog.LevelWarn - } - s.logger.Log(ctx, level, "mirror schedule run", - "schedule", index, "mirrored", progress.Completed, "unchanged", progress.Skipped, - "failed", progress.Failed, "bytes", progress.Bytes, "duration", time.Since(started).Round(time.Millisecond)) -} - -// scheduleDelay spreads runs so instances and schedules started together -// don't hit the registry at once. The first run comes within ten minutes, so -// a restart doesn't push a refresh back by a whole interval. -func scheduleDelay(interval time.Duration, first bool, randN func(int64) int64) time.Duration { - if first { - return time.Duration(randN(int64(min(interval, maxFirstScheduleDelay)))) - } - spread := interval / scheduleJitter - return interval - spread + time.Duration(randN(int64(2*spread)+1)) //nolint:mnd // spread on both sides -} - -// scheduleKey names a schedule by what it mirrors and how often, so -// instances with the same configuration agree on it and a changed schedule -// does not inherit the old one's last run. -func scheduleKey(schedule Schedule) string { - h := sha256.New() - for _, purl := range schedule.PURLs { - _, _ = fmt.Fprintf(h, "purl %s\n", purl) - } - for _, platform := range schedule.Platforms { - _, _ = fmt.Fprintf(h, "platform %s\n", platform) - } - _, _ = fmt.Fprintf(h, "interval %s\n", schedule.Interval) - return "mirror-schedule:" + hex.EncodeToString(h.Sum(nil))[:32] -} diff --git a/internal/mirror/schedule_test.go b/internal/mirror/schedule_test.go deleted file mode 100644 index b5b0143..0000000 --- a/internal/mirror/schedule_test.go +++ /dev/null @@ -1,281 +0,0 @@ -package mirror - -import ( - "context" - "errors" - "io" - "log/slog" - "math/rand/v2" - "reflect" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/handler" -) - -func TestScheduleDelay(t *testing.T) { - lowest := func(int64) int64 { return 0 } - highest := func(n int64) int64 { return n - 1 } - tests := []struct { - name string - interval time.Duration - first bool - randN func(int64) int64 - want time.Duration - }{ - {name: "first run lowest", interval: 6 * time.Hour, first: true, randN: lowest, want: 0}, - {name: "first run capped at ten minutes", interval: 6 * time.Hour, first: true, randN: highest, want: 10*time.Minute - 1}, - {name: "first run within a short interval", interval: 2 * time.Minute, first: true, randN: highest, want: 2*time.Minute - 1}, - {name: "later run lowest", interval: 6 * time.Hour, randN: lowest, want: 6*time.Hour - 36*time.Minute}, - {name: "later run highest", interval: 6 * time.Hour, randN: highest, want: 6*time.Hour + 36*time.Minute}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := scheduleDelay(tt.interval, tt.first, tt.randN); got != tt.want { - t.Errorf("scheduleDelay() = %v, want %v", got, tt.want) - } - }) - } -} - -// scheduleImages records MirrorImage calls and can block or fail them. -type scheduleImages struct { - mu sync.Mutex - refs []handler.ImageRef - platforms [][]handler.Platform - calls atomic.Int32 - running atomic.Int32 - overlap atomic.Bool - block chan struct{} - err error -} - -func (f *scheduleImages) MirrorImage(ctx context.Context, ref handler.ImageRef, platforms []handler.Platform) (handler.ImageResult, error) { - if f.running.Add(1) > 1 { - f.overlap.Store(true) - } - defer f.running.Add(-1) - f.mu.Lock() - f.refs = append(f.refs, ref) - f.platforms = append(f.platforms, platforms) - f.mu.Unlock() - f.calls.Add(1) - if f.block != nil { - select { - case <-f.block: - case <-ctx.Done(): - return handler.ImageResult{}, ctx.Err() - } - } - return handler.ImageResult{}, f.err -} - -func newTestScheduler(images ImageMirrorer, schedules ...Schedule) *Scheduler { - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - m := New(nil, nil, nil, logger, 1) - m.Images = images - s := NewScheduler(m, schedules, logger) - s.randN = func(int64) int64 { return 0 } - return s -} - -func waitFor(t *testing.T, what string, cond func() bool) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for !cond() { - if time.Now().After(deadline) { - t.Fatalf("timed out waiting for %s", what) - } - time.Sleep(time.Millisecond) - } -} - -func TestSchedulerRepeatsRunsAndStops(t *testing.T) { - images := &scheduleImages{err: errors.New("upstream down")} - s := newTestScheduler(images, Schedule{ - PURLs: []string{"pkg:docker/library/alpine@3.20"}, - Platforms: []string{"linux/arm64"}, - Interval: time.Millisecond, - }) - ctx, cancel := context.WithCancel(context.Background()) - s.Start(ctx) - // A failing run must not end the schedule. - waitFor(t, "three runs", func() bool { return images.calls.Load() >= 3 }) - cancel() - s.Wait() - - images.mu.Lock() - defer images.mu.Unlock() - wantRef := handler.ImageRef{Name: "library/alpine", References: []string{"3.20"}} - if !reflect.DeepEqual(images.refs[0], wantRef) { - t.Errorf("ref = %+v, want %+v", images.refs[0], wantRef) - } - wantPlatforms := []handler.Platform{{OS: "linux", Architecture: "arm64"}} - if !reflect.DeepEqual(images.platforms[0], wantPlatforms) { - t.Errorf("platforms = %+v, want %+v", images.platforms[0], wantPlatforms) - } -} - -func TestSchedulerRunsDoNotOverlap(t *testing.T) { - images := &scheduleImages{block: make(chan struct{})} - s := newTestScheduler(images, Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Interval: time.Millisecond}) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s.Start(ctx) - - waitFor(t, "first run", func() bool { return images.calls.Load() == 1 }) - // Many intervals pass while the run blocks; none may start a second run. - time.Sleep(50 * time.Millisecond) - if got := images.calls.Load(); got != 1 { - t.Fatalf("calls while the first run blocks = %d, want 1", got) - } - images.block <- struct{}{} - waitFor(t, "second run", func() bool { return images.calls.Load() == 2 }) - cancel() - s.Wait() - if images.overlap.Load() { - t.Error("two runs of one schedule overlapped") - } -} - -func TestSchedulerWaitReturnsAfterCancelDuringRun(t *testing.T) { - images := &scheduleImages{block: make(chan struct{})} - s := newTestScheduler(images, - Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Interval: time.Hour}, - Schedule{PURLs: []string{"pkg:docker/nginx@stable"}, Interval: time.Hour}) - ctx, cancel := context.WithCancel(context.Background()) - s.Start(ctx) - waitFor(t, "both schedules running", func() bool { return images.calls.Load() == 2 }) - - cancel() - done := make(chan struct{}) - go func() { - s.Wait() - close(done) - }() - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("Wait did not return after cancel") - } -} - -func TestSchedulerSkipsScheduleWithoutInterval(t *testing.T) { - images := &scheduleImages{} - s := newTestScheduler(images, Schedule{PURLs: []string{"pkg:docker/alpine@3"}}) - // The real random source panics on a zero interval. - s.randN = rand.Int64N - ctx, cancel := context.WithCancel(context.Background()) - s.Start(ctx) - cancel() - s.Wait() - if got := images.calls.Load(); got != 0 { - t.Errorf("calls = %d, want 0", got) - } -} - -func TestSchedulerSkipsRunClaimedElsewhere(t *testing.T) { - images := &scheduleImages{} - schedule := Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Interval: time.Hour} - s := newTestScheduler(images, schedule) - var mu sync.Mutex - var keys []string - var gaps []time.Duration - claim := true - s.Claim = func(key string, minGap time.Duration) (bool, error) { - mu.Lock() - defer mu.Unlock() - keys = append(keys, key) - gaps = append(gaps, minGap) - return claim, nil - } - - s.run(context.Background(), 0, schedule) - if got := images.calls.Load(); got != 1 { - t.Fatalf("calls after a claimed run = %d, want 1", got) - } - claim = false - s.run(context.Background(), 0, schedule) - if got := images.calls.Load(); got != 1 { - t.Fatalf("calls after a run claimed elsewhere = %d, want 1", got) - } - s.Claim = func(string, time.Duration) (bool, error) { return false, errors.New("database down") } - s.run(context.Background(), 0, schedule) - if got := images.calls.Load(); got != 1 { - t.Fatalf("calls after a failed claim = %d, want 1", got) - } - - if keys[0] != keys[1] { - t.Errorf("claim keys differ between runs: %q, %q", keys[0], keys[1]) - } - // Runs come at least 0.9 intervals apart; the gap stays below that. - if want := 48 * time.Minute; gaps[0] != want { - t.Errorf("minGap = %s, want %s", gaps[0], want) - } -} - -// TestSchedulerReleasesInterruptedRun checks that a run cut off by shutdown -// does not count as done: the next attempt, on a restarted instance or -// another one, runs the schedule again. -func TestSchedulerReleasesInterruptedRun(t *testing.T) { - images := &scheduleImages{block: make(chan struct{})} - schedule := Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Interval: time.Hour} - s := newTestScheduler(images, schedule) - // The claims of a shared database: a claimed key is not claimed again. - var mu sync.Mutex - claims := map[string]bool{} - s.Claim = func(key string, _ time.Duration) (bool, error) { - mu.Lock() - defer mu.Unlock() - if claims[key] { - return false, nil - } - claims[key] = true - return true, nil - } - s.Release = func(key string) error { - mu.Lock() - defer mu.Unlock() - delete(claims, key) - return nil - } - - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - s.run(ctx, 0, schedule) - close(done) - }() - waitFor(t, "the run to start", func() bool { return images.calls.Load() == 1 }) - cancel() - <-done - - close(images.block) - s.run(context.Background(), 0, schedule) - if got := images.calls.Load(); got != 2 { - t.Fatalf("calls = %d, want 2: the interrupted run blocked the next one", got) - } - // A finished run keeps its claim. - s.run(context.Background(), 0, schedule) - if got := images.calls.Load(); got != 2 { - t.Fatalf("calls = %d, want 2 after a finished run", got) - } -} - -func TestScheduleKey(t *testing.T) { - base := Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Platforms: []string{"linux/amd64"}, Interval: time.Hour} - if scheduleKey(base) != scheduleKey(Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Platforms: []string{"linux/amd64"}, Interval: time.Hour}) { - t.Error("same schedule gave different keys") - } - for name, other := range map[string]Schedule{ - "purls": {PURLs: []string{"pkg:docker/alpine@4"}, Platforms: base.Platforms, Interval: base.Interval}, - "platforms": {PURLs: base.PURLs, Platforms: []string{"linux/arm64"}, Interval: base.Interval}, - "interval": {PURLs: base.PURLs, Platforms: base.Platforms, Interval: 2 * time.Hour}, - } { - if scheduleKey(other) == scheduleKey(base) { - t.Errorf("changing %s kept the key", name) - } - } -} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index c0441c4..c583fb7 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -1,14 +1,19 @@ package mirror import ( + "bytes" "context" + "encoding/json" "fmt" - "strings" + "os" - "github.com/git-pkgs/proxy/internal/packageurl" + cdx "github.com/CycloneDX/cyclonedx-go" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" - "github.com/git-pkgs/sbom" + _ "github.com/git-pkgs/registries/all" + spdxjson "github.com/spdx/tools-golang/json" + "github.com/spdx/tools-golang/spdx" + spdxtv "github.com/spdx/tools-golang/tagvalue" ) // PackageVersion identifies a specific package version to mirror. @@ -16,16 +21,6 @@ type PackageVersion struct { Ecosystem string Name string Version string - // Namespace is the PURL namespace of apk packages, which Name leaves - // out. It picks the upstream.apk repository. - Namespace string - // Qualifiers are the PURL qualifiers. Images use repository_url, tag and - // arch; PyPI uses file_name, and conda subdir, build and type, to select - // which files of the version are mirrored. Julia uses uuid to identify - // the package. - Qualifiers map[string]string - // Platforms limit which platforms of an image index are mirrored. - Platforms []string } func (pv PackageVersion) String() string { @@ -38,23 +33,16 @@ type Source interface { } // PURLSource yields packages from PURL strings. -// Versioned PURLs produce a single item. Unversioned PURLs produce one item -// per version Versions lists, or only the newest with Latest. +// Versioned PURLs produce a single item. Unversioned PURLs look up all versions from the registry. type PURLSource struct { - PURLs []string - // Versions lists the versions of unversioned PURLs. Nil lists them from - // the public registries. - Versions VersionLister - // Latest mirrors only the newest version of unversioned PURLs. - Latest bool - // Platforms limit which platforms of container images are mirrored. - Platforms []string + PURLs []string + RegClient *registries.Client } func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) error) error { - versions := s.Versions - if versions == nil { - versions = RegistryVersions{} + client := s.RegClient + if client == nil { + client = registries.DefaultClient() } for _, purlStr := range s.PURLs { @@ -63,40 +51,26 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("parsing PURL %q: %w", purlStr, err) } - ecosystem, name, qualifiers := mirrorTarget(p) - var namespace string - if ecosystem == "alpine" { - namespace = p.Namespace + ecosystem := purl.PURLTypeToEcosystem(p.Type) + name := p.Name + if p.Namespace != "" { + name = p.Namespace + "/" + p.Name } - // Images are never expanded to all tags, and conda packages without - // a channel aren't listed; both fail when they are mirrored. A Git - // repository without a commit is mirrored as a whole. - if p.Version != "" || isImageEcosystem(ecosystem) || ecosystem == "git" || (ecosystem == "conda" && !strings.Contains(name, "/")) { - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Namespace: namespace, Qualifiers: qualifiers} - if isImageEcosystem(ecosystem) { - pv.Qualifiers = p.Qualifiers.Map() - } - if takesPlatforms(ecosystem) { - pv.Platforms = s.Platforms - } - if err := fn(pv); err != nil { + if p.Version != "" { + if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version}); err != nil { return err } continue } - // Unversioned: enumerate all versions, or the newest - listed, err := s.listVersions(ctx, versions, ecosystem, name) + // Unversioned: enumerate all versions + versions, err := s.fetchVersions(ctx, client, ecosystem, name) if err != nil { return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } - for _, v := range listed { - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Namespace: namespace, Qualifiers: qualifiers} - if takesPlatforms(ecosystem) { - pv.Platforms = s.Platforms - } - if err := fn(pv); err != nil { + for _, v := range versions { + if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v}); err != nil { return err } } @@ -104,102 +78,113 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return nil } -func (s *PURLSource) listVersions(ctx context.Context, versions VersionLister, ecosystem, name string) ([]string, error) { - if !s.Latest { - return versions.ListVersions(ctx, ecosystem, name) +func (s *PURLSource) fetchVersions(ctx context.Context, client *registries.Client, ecosystem, name string) ([]string, error) { + reg, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", client) + if err != nil { + return nil, err } - var latest string - if finder, ok := versions.(LatestFinder); ok { - v, err := finder.LatestVersion(ctx, ecosystem, name) - if err != nil { - return nil, err - } - latest = v - } else { - listed, err := versions.ListVersions(ctx, ecosystem, name) - if err != nil { - return nil, err - } - all := make([]registries.Version, len(listed)) - for i, v := range listed { - all[i] = registries.Version{Number: v} - } - if v := registries.SelectLatestVersion(all, purl.EcosystemToPURLType(ecosystem), ""); v != nil { - latest = v.Number - } + versions, err := reg.FetchVersions(ctx, name) + if err != nil { + return nil, err } - if latest == "" { - return nil, nil + result := make([]string, len(versions)) + for i, v := range versions { + result[i] = v.Number } - return []string{latest}, nil + return result, nil } -// mirrorTarget returns the ecosystem, name and file qualifiers a PURL is -// mirrored under. -func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]string) { - ecosystem, name = packageurl.HandlerPackage(p) - qualifiers = fileQualifiers(ecosystem, p.Qualifiers.Map()) - switch ecosystem { - case "julia": - qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} - case "alpine": - qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro")} - } - return ecosystem, name, qualifiers -} - -// SBOMSource extracts package versions from CycloneDX or SPDX SBOM data. +// SBOMSource extracts package versions from a CycloneDX or SPDX SBOM file. type SBOMSource struct { - Data []byte - Name string - // Versions lists the versions of unversioned PURLs. Nil lists them from - // the public registries. - Versions VersionLister - // Latest mirrors only the newest version of unversioned PURLs. - Latest bool - // Platforms limit which platforms of container images are mirrored. - Platforms []string + Path string + RegClient *registries.Client } func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) error) error { purls, err := s.extractPURLs() if err != nil { - if s.Name != "" { - return fmt.Errorf("parsing SBOM %s: %w", s.Name, err) - } - return fmt.Errorf("parsing SBOM: %w", err) + return fmt.Errorf("reading SBOM %s: %w", s.Path, err) } - inner := &PURLSource{PURLs: purls, Versions: s.Versions, Latest: s.Latest, Platforms: s.Platforms} + inner := &PURLSource{PURLs: purls, RegClient: s.RegClient} return inner.Enumerate(ctx, fn) } func (s *SBOMSource) extractPURLs() ([]string, error) { - doc, err := sbom.Parse(s.Data) + data, err := os.ReadFile(s.Path) if err != nil { return nil, err } - var purls []string - for _, p := range doc.Packages { - for _, ref := range p.ExternalRefs { - if ref.Type != "purl" || ref.Locator == "" { - continue - } - locator := ref.Locator - if p.Version != "" { - parsed, err := purl.Parse(locator) - if err != nil { - return nil, fmt.Errorf("parsing PURL %q: %w", locator, err) - } - if parsed.Version == "" { - locator = parsed.WithVersion(p.Version).String() - } - } - purls = append(purls, locator) + + // Try CycloneDX first + if purls, err := extractCycloneDXPURLs(data); err == nil && len(purls) > 0 { + return purls, nil + } + + // Try SPDX JSON + if purls, err := extractSPDXJSONPURLs(data); err == nil && len(purls) > 0 { + return purls, nil + } + + // Try SPDX tag-value + if purls, err := extractSPDXTVPURLs(data); err == nil && len(purls) > 0 { + return purls, nil + } + + return nil, fmt.Errorf("could not parse SBOM as CycloneDX or SPDX") +} + +func extractCycloneDXPURLs(data []byte) ([]string, error) { + bom := new(cdx.BOM) + if err := json.Unmarshal(data, bom); err != nil { + // Try XML + decoder := cdx.NewBOMDecoder(bytes.NewReader(data), cdx.BOMFileFormatXML) + bom = new(cdx.BOM) + if err := decoder.Decode(bom); err != nil { + return nil, err } } - if len(purls) == 0 { - return nil, fmt.Errorf("SBOM contains no package URLs") + + if bom.Components == nil { + return nil, nil + } + + var purls []string + for _, c := range *bom.Components { + if c.PackageURL != "" { + purls = append(purls, c.PackageURL) + } } return purls, nil } + +func extractSPDXJSONPURLs(data []byte) ([]string, error) { + doc, err := spdxjson.Read(bytes.NewReader(data)) + if err != nil { + return nil, err + } + return extractSPDXDocPURLs(doc), nil +} + +func extractSPDXTVPURLs(data []byte) ([]string, error) { + doc, err := spdxtv.Read(bytes.NewReader(data)) + if err != nil { + return nil, err + } + return extractSPDXDocPURLs(doc), nil +} + +func extractSPDXDocPURLs(doc *spdx.Document) []string { + if doc == nil { + return nil + } + var purls []string + for _, pkg := range doc.Packages { + for _, ref := range pkg.PackageExternalReferences { + if ref.RefType == "purl" { + purls = append(purls, ref.Locator) + } + } + } + return purls +} diff --git a/internal/mirror/source_formats_test.go b/internal/mirror/source_formats_test.go deleted file mode 100644 index 972c047..0000000 --- a/internal/mirror/source_formats_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package mirror - -import ( - "context" - "reflect" - "testing" -) - -func TestSBOMSourcePreservesPackageIdentities(t *testing.T) { - for name, data := range map[string]string{ - "CycloneDX JSON": `{"components":[ - {"name":"shared","version":"1.0","purl":"pkg:npm/%40alpha/shared@1.0"}, - {"name":"shared","version":"1.0","purl":"pkg:npm/%40beta/shared@1.0"}]}`, - "CycloneDX XML": ` - shared1.0pkg:npm/%40alpha/shared@1.0 - shared1.0pkg:npm/%40beta/shared@1.0 - `, - "SPDX JSON": `{"spdxVersion":"SPDX-2.3","packages":[{"name":"shared","versionInfo":"1.0","externalRefs":[ - {"referenceType":"purl","referenceLocator":"pkg:npm/%40alpha/shared@1.0"}, - {"referenceType":"purl","referenceLocator":"pkg:npm/%40beta/shared@1.0"}]}]}`, - "SPDX tag-value": `DataLicense: CC0-1.0 -SPDXVersion: SPDX-2.3 -SPDXID: SPDXRef-DOCUMENT -DocumentName: example -DocumentNamespace: https://example.com/sbom -PackageName: shared -SPDXID: SPDXRef-shared -PackageVersion: 1.0 -ExternalRef: PACKAGE-MANAGER purl pkg:npm/%40alpha/shared@1.0 -ExternalRef: PACKAGE-MANAGER purl pkg:npm/%40beta/shared@1.0 -`, - } { - t.Run(name, func(t *testing.T) { - source := &SBOMSource{Data: []byte(data), Name: name} - var got []PackageVersion - err := source.Enumerate(context.Background(), func(pv PackageVersion) error { - got = append(got, pv) - return nil - }) - if err != nil { - t.Fatal(err) - } - want := []PackageVersion{ - {Ecosystem: "npm", Name: "@alpha/shared", Version: "1.0"}, - {Ecosystem: "npm", Name: "@beta/shared", Version: "1.0"}, - } - if !reflect.DeepEqual(got, want) { - t.Errorf("Enumerate = %+v, want %+v", got, want) - } - }) - } -} - -func TestSBOMSourcePreservesImageOptions(t *testing.T) { - source := &SBOMSource{ - Data: []byte(`{"bomFormat":"CycloneDX","components":[{"name":"alpine","purl":"pkg:docker/library/alpine@3.20?repository_url=ghcr.io&arch=arm64"}]}`), - Platforms: []string{"linux/amd64"}, - } - var got []PackageVersion - if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { - got = append(got, pv) - return nil - }); err != nil { - t.Fatal(err) - } - want := []PackageVersion{{ - Ecosystem: "docker", Name: "library/alpine", Version: "3.20", - Qualifiers: map[string]string{"repository_url": "ghcr.io", "arch": "arm64"}, - Platforms: []string{"linux/amd64"}, - }} - if !reflect.DeepEqual(got, want) { - t.Errorf("Enumerate = %+v, want %+v", got, want) - } -} diff --git a/internal/mirror/source_test.go b/internal/mirror/source_test.go index faddf35..b0bb1be 100644 --- a/internal/mirror/source_test.go +++ b/internal/mirror/source_test.go @@ -3,8 +3,8 @@ package mirror import ( "context" "encoding/json" - "slices" - "strings" + "os" + "path/filepath" "testing" ) @@ -116,7 +116,8 @@ func TestSBOMSourceCycloneDXJSON(t *testing.T) { }, } - source := &SBOMSource{Data: marshalJSON(t, bom)} + path := writeTempJSON(t, bom) + source := &SBOMSource{Path: path} var items []PackageVersion err := source.Enumerate(context.Background(), func(pv PackageVersion) error { @@ -163,7 +164,8 @@ func TestSBOMSourceSPDXJSON(t *testing.T) { }, } - source := &SBOMSource{Data: marshalJSON(t, doc)} + path := writeTempJSON(t, doc) + source := &SBOMSource{Path: path} var items []PackageVersion err := source.Enumerate(context.Background(), func(pv PackageVersion) error { @@ -183,28 +185,30 @@ func TestSBOMSourceSPDXJSON(t *testing.T) { } } -func TestSBOMSourceEmptyData(t *testing.T) { - source := &SBOMSource{} +func TestSBOMSourceNonexistentFile(t *testing.T) { + source := &SBOMSource{Path: "/nonexistent/sbom.json"} err := source.Enumerate(context.Background(), func(pv PackageVersion) error { return nil }) if err == nil { - t.Fatal("expected error for empty SBOM") + t.Fatal("expected error for nonexistent file") } } func TestSBOMSourceInvalidFormat(t *testing.T) { - source := &SBOMSource{Data: []byte("this is not an SBOM"), Name: "invalid.sbom"} + path := filepath.Join(t.TempDir(), "invalid.txt") + if err := os.WriteFile(path, []byte("this is not an SBOM"), 0644); err != nil { + t.Fatal(err) + } + + source := &SBOMSource{Path: path} err := source.Enumerate(context.Background(), func(pv PackageVersion) error { return nil }) if err == nil { t.Fatal("expected error for invalid SBOM") } - if !strings.Contains(err.Error(), "invalid.sbom") { - t.Errorf("error = %q, want source name", err) - } } func TestSBOMSourceEmptyCycloneDX(t *testing.T) { @@ -212,7 +216,11 @@ func TestSBOMSourceEmptyCycloneDX(t *testing.T) { "bomFormat": "CycloneDX", "specVersion": "1.4", } - source := &SBOMSource{Data: marshalJSON(t, bom)} + path := writeTempJSON(t, bom) + + // This should fall through to SPDX parsing, which will also fail, + // resulting in an error about not being able to parse + source := &SBOMSource{Path: path} err := source.Enumerate(context.Background(), func(pv PackageVersion) error { return nil }) @@ -221,47 +229,15 @@ func TestSBOMSourceEmptyCycloneDX(t *testing.T) { } } -func marshalJSON(t *testing.T, v any) []byte { +func writeTempJSON(t *testing.T, v any) string { t.Helper() data, err := json.Marshal(v) if err != nil { t.Fatal(err) } - return data -} - -// The mirror files packages under the ecosystem and package names the -// handlers use, so a mirrored package and the same package cached by a client -// share one name. -func TestPURLSourceUsesHandlerEcosystemNames(t *testing.T) { - source := &PURLSource{PURLs: []string{ - "pkg:gem/rake@13.2.1", - "pkg:composer/psr/log@3.0.2", - "pkg:apk/alpine/curl@8.14.1-r2", - "pkg:npm/left-pad@1.3.0", - "pkg:maven/org.slf4j/slf4j-api@2.0.13", - "pkg:pypi/Zope.Interface@6.4", - "pkg:nuget/Newtonsoft.Json@13.0.3", - }} - - var got []string - if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { - got = append(got, pv.Ecosystem+" "+pv.Name) - return nil - }); err != nil { - t.Fatalf("Enumerate() error = %v", err) - } - - want := []string{ - "gem rake", - "composer psr/log", - "alpine curl", - "npm left-pad", - "maven org.slf4j:slf4j-api", - "pypi zope-interface", - "nuget newtonsoft.json", - } - if !slices.Equal(got, want) { - t.Errorf("packages = %q, want %q", got, want) - } + path := filepath.Join(t.TempDir(), "sbom.json") + if err := os.WriteFile(path, data, 0644); err != nil { + t.Fatal(err) + } + return path } diff --git a/internal/mirror/swift.go b/internal/mirror/swift.go deleted file mode 100644 index 153bb10..0000000 --- a/internal/mirror/swift.go +++ /dev/null @@ -1,57 +0,0 @@ -package mirror - -import ( - "context" - "fmt" - "strings" - - "github.com/git-pkgs/proxy/internal/handler" -) - -// mirrorSwift mirrors the source archive of a Swift registry release, with -// its release metadata, from the registry the proxy serves Swift packages -// from. -func (m *Mirror) mirrorSwift(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - upstream, err := m.registryURL("swift", pv.Name) - if err != nil { - m.recordFailure(tracker, pv, err) - return - } - h := handler.NewSwiftHandler(m.proxy, "", upstream) - var scope, name string - if source := handler.SwiftSourceURL(pv.Name); source != "" { - scope, name, err = h.ResolveIdentity(ctx, source) - } else { - scope, name, err = swiftScopeAndName(pv.Name) - } - if err != nil { - m.recordFailure(tracker, pv, err) - return - } - result, err := h.MirrorRelease(ctx, scope, name, pv.Version) - if err != nil { - m.recordFailure(tracker, pv, err) - return - } - m.recordResult(tracker, pv, result) -} - -// registryURL returns the registry base URL Versions lists purlType packages -// from, which is the configured upstream, or empty for the default one. -func (m *Mirror) registryURL(purlType, name string) (string, error) { - if r, ok := m.Versions.(RegistryVersions); ok && r.URL != nil { - return r.URL(purlType, name) - } - return "", nil -} - -// swiftScopeAndName splits the name of a pkg:swift PURL whose namespace is -// a registry scope, as in pkg:swift/apple/swift-nio for the identity -// apple.swift-nio. -func swiftScopeAndName(name string) (scope, pkg string, _ error) { - scope, pkg, ok := strings.Cut(name, "/") - if !ok || scope == "" || pkg == "" || strings.ContainsAny(scope, ".:") || strings.Contains(pkg, "/") { - return "", "", fmt.Errorf("pkg:swift/%s needs a registry scope or a source repository as the namespace, such as pkg:swift/apple/swift-nio or pkg:swift/github.com/apple/swift-nio", name) - } - return scope, pkg, nil -} diff --git a/internal/mirror/swift_test.go b/internal/mirror/swift_test.go deleted file mode 100644 index 4ae0764..0000000 --- a/internal/mirror/swift_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package mirror - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - - "github.com/git-pkgs/registries" -) - -func TestSwiftIdentity(t *testing.T) { - registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/identifiers" || r.Header.Get("Accept") != "application/vnd.swift.registry.v1+json" { - http.Error(w, "unexpected request", http.StatusBadRequest) - return - } - switch r.URL.Query().Get("url") { - case "https://github.com/apple/swift-nio": - _, _ = w.Write([]byte(`{"identifiers":["apple.swift-nio","mirror.swift-nio"]}`)) - case "https://github.com/acme/empty": - _, _ = w.Write([]byte(`{"identifiers":[]}`)) - default: - http.NotFound(w, r) - } - })) - defer registry.Close() - - for name, want := range map[string]string{ - "apple/swift-nio": "apple.swift-nio", - "github.com/apple/swift-nio": "apple.swift-nio", - "github.com/acme/empty": "", - "github.com/acme/unknown": "", - "swift-nio": "", - "apple/": "", - "apple/swift-nio/extra": "", - } { - got, err := swiftIdentity(context.Background(), registries.DefaultClient(), registry.URL, name) - if got != want || (want == "") != (err != nil) { - t.Errorf("swiftIdentity(%q) = %q, %v; want %q", name, got, err, want) - } - } -} diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go deleted file mode 100644 index 0a7e1ba..0000000 --- a/internal/mirror/versions.go +++ /dev/null @@ -1,174 +0,0 @@ -package mirror - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "strings" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/handler" - "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries" - _ "github.com/git-pkgs/registries/all" -) - -// VersionLister lists the versions of a package, for PURLs without one. -type VersionLister interface { - ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) -} - -// RegistryVersions lists versions through the registries clients. -type RegistryVersions struct { - // Client makes the requests. Nil uses registries.DefaultClient. - Client *registries.Client - // URL returns the registry base URL to list a package from, by PURL - // type. Nil, or an empty URL, uses the public registry. - URL func(purlType, name string) (string, error) -} - -// VersionFinder finds one version of a package, with the files it -// publishes, for ecosystems the mirror fetches from the version's metadata. -type VersionFinder interface { - FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) -} - -// LatestFinder picks the latest version of a package, for unversioned PURLs -// when only the latest is mirrored. Listers without it fall back to -// registries.SelectLatestVersion over ListVersions. -type LatestFinder interface { - LatestVersion(ctx context.Context, ecosystem, name string) (string, error) -} - -// LatestVersion implements LatestFinder with registries.FetchLatestVersion: -// the version the registry advertises as latest where it has one, such as -// npm's dist-tags.latest, and otherwise the newest version by publish time -// that isn't yanked, retracted or deprecated. -func (r RegistryVersions) LatestVersion(ctx context.Context, ecosystem, name string) (string, error) { - reg, name, err := r.registryFor(ctx, ecosystem, name) - if err != nil { - return "", err - } - v, err := registries.FetchLatestVersion(ctx, reg, name) - if err != nil || v == nil { - return "", err - } - return v.Number, nil -} - -// ListVersions implements VersionLister. -func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) { - versions, err := r.fetchVersions(ctx, ecosystem, name) - if err != nil { - return nil, err - } - result := make([]string, len(versions)) - for i, v := range versions { - result[i] = v.Number - } - return result, nil -} - -// FindVersion implements VersionFinder. -func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) { - versions, err := r.fetchVersions(ctx, ecosystem, name) - if err != nil { - return registries.Version{}, err - } - for _, v := range versions { - if v.Number == version { - return v, nil - } - } - return registries.Version{}, fmt.Errorf("%w: %s %s@%s", registries.ErrNotFound, ecosystem, name, version) -} - -func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { - reg, name, err := r.registryFor(ctx, ecosystem, name) - if err != nil { - return nil, err - } - return reg.FetchVersions(ctx, name) -} - -// registryFor returns the client for the registry name is listed in, and -// the name the registry knows the package by. -func (r RegistryVersions) registryFor(ctx context.Context, ecosystem, name string) (registries.Registry, string, error) { //nolint:ireturn // registries.New returns the interface - reg, err := r.registry(ecosystem, name) - if err != nil { - return nil, "", err - } - switch purl.EcosystemToPURLType(ecosystem) { - case "helm": - // The namespace names the repository, which URL resolves to the - // index the chart is listed in. - _, name, _ = strings.Cut(name, "/") - case "swift": - var baseURL string - if baseURL, err = r.baseURL("swift", name); err != nil { - return nil, "", err - } - if name, err = swiftIdentity(ctx, r.client(), baseURL, name); err != nil { - return nil, "", err - } - } - return reg, name, nil -} - -// registry returns the client for the registry name is listed in. -func (r RegistryVersions) registry(ecosystem, name string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface - purlType := purl.EcosystemToPURLType(ecosystem) - baseURL, err := r.baseURL(purlType, name) - if err != nil { - return nil, err - } - return registries.New(purlType, baseURL, r.client()) -} - -func (r RegistryVersions) baseURL(purlType, name string) (string, error) { - if r.URL == nil { - return "", nil - } - return r.URL(purlType, name) -} - -func (r RegistryVersions) client() *registries.Client { - if r.Client == nil { - return registries.DefaultClient() - } - return r.Client -} - -// swiftIdentity returns the registry identity, scope.name, of the name of a -// pkg:swift PURL, looking a source repository up on the registry at baseURL. -func swiftIdentity(ctx context.Context, client *registries.Client, baseURL, name string) (string, error) { - source := handler.SwiftSourceURL(name) - if source == "" { - scope, pkg, err := swiftScopeAndName(name) - if err != nil { - return "", err - } - return scope + "." + pkg, nil - } - if baseURL == "" { - baseURL = config.DefaultSwiftUpstream - } - var body json.RawMessage - _, err := client.GetJSONWithHeaders(ctx, - strings.TrimSuffix(baseURL, "/")+"/identifiers?"+url.Values{"url": {source}}.Encode(), - http.Header{"Accept": {"application/vnd.swift.registry.v1+json"}}, &body) - if httpErr := (*registries.HTTPError)(nil); errors.As(err, &httpErr) && httpErr.IsNotFound() { - return "", fmt.Errorf("the Swift registry has no package for %s", source) - } - if err != nil { - return "", err - } - scope, pkg, err := handler.SwiftIdentityOf(body, source) - if err != nil { - return "", err - } - return scope + "." + pkg, nil -} diff --git a/internal/mirror/versions_test.go b/internal/mirror/versions_test.go deleted file mode 100644 index 29f9f41..0000000 --- a/internal/mirror/versions_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package mirror - -import ( - "context" - "encoding/json" - "errors" - "reflect" - "testing" -) - -// fakeVersions lists fixed versions and records what it was asked for. -type fakeVersions struct { - versions []string - err error - asked []string -} - -func (f *fakeVersions) ListVersions(_ context.Context, ecosystem, name string) ([]string, error) { - f.asked = append(f.asked, ecosystem+"/"+name) - return f.versions, f.err -} - -func enumerate(t *testing.T, source Source) []PackageVersion { - t.Helper() - var items []PackageVersion - if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { - items = append(items, pv) - return nil - }); err != nil { - t.Fatalf("Enumerate() error = %v", err) - } - return items -} - -func TestPURLSourceListsVersionsThroughLister(t *testing.T) { - versions := &fakeVersions{versions: []string{"1.0.0", "1.1.0"}} - source := &PURLSource{PURLs: []string{"pkg:npm/%40acme/widget"}, Versions: versions} - - items := enumerate(t, source) - - want := []PackageVersion{ - {Ecosystem: "npm", Name: "@acme/widget", Version: "1.0.0"}, - {Ecosystem: "npm", Name: "@acme/widget", Version: "1.1.0"}, - } - if !reflect.DeepEqual(items, want) { - t.Errorf("items = %v, want %v", items, want) - } - if !reflect.DeepEqual(versions.asked, []string{"npm/@acme/widget"}) { - t.Errorf("lister asked for %v, want npm/@acme/widget", versions.asked) - } -} - -func TestPURLSourceReportsListerError(t *testing.T) { - source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{err: errors.New("upstream down")}} - err := source.Enumerate(context.Background(), func(PackageVersion) error { return nil }) - if err == nil { - t.Fatal("Enumerate() succeeded although listing versions failed") - } -} - -func TestSBOMSourceListsVersionsThroughLister(t *testing.T) { - bom := map[string]any{ - "bomFormat": "CycloneDX", - "specVersion": "1.4", - "components": []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}}, - } - versions := &fakeVersions{versions: []string{"1.3.0"}} - source := &SBOMSource{Data: marshalJSON(t, bom), Versions: versions} - - items := enumerate(t, source) - - if len(items) != 1 || items[0].Version != "1.3.0" { - t.Errorf("items = %v, want left-pad@1.3.0 from the lister", items) - } -} - -func TestSourceFromRequestUsesMirrorVersions(t *testing.T) { - m := setupTestMirror(t, 1) - versions := &fakeVersions{} - m.Versions = versions - js := NewJobStore(context.Background(), m) - - source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}}) - if err != nil { - t.Fatalf("sourceFromRequest() error = %v", err) - } - if got := source.(*PURLSource).Versions; got != versions { - t.Errorf("PURL source lists versions through %v, want the mirror's lister", got) - } - - source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`)}) - if err != nil { - t.Fatalf("sourceFromRequest() error = %v", err) - } - if got := source.(*SBOMSource).Versions; got != versions { - t.Errorf("SBOM source lists versions through %v, want the mirror's lister", got) - } -} - -func TestPURLSourceKeepsJuliaUUID(t *testing.T) { - const uuid = "7876af07-990d-54b4-ab0e-23690620f79a" - versions := &fakeVersions{versions: []string{"0.5.5"}} - source := &PURLSource{ - PURLs: []string{"pkg:julia/Example@0.5.4?uuid=" + uuid, "pkg:julia/Example?uuid=" + uuid}, - Versions: versions, - } - - items := enumerate(t, source) - - qualifiers := map[string]string{"uuid": uuid} - want := []PackageVersion{ - {Ecosystem: "julia", Name: "Example", Version: "0.5.4", Qualifiers: qualifiers}, - {Ecosystem: "julia", Name: "Example", Version: "0.5.5", Qualifiers: qualifiers}, - } - if !reflect.DeepEqual(items, want) { - t.Errorf("items = %v, want %v", items, want) - } -} diff --git a/internal/packageurl/handler.go b/internal/packageurl/handler.go deleted file mode 100644 index 5edcf80..0000000 --- a/internal/packageurl/handler.go +++ /dev/null @@ -1,70 +0,0 @@ -package packageurl - -import ( - "github.com/git-pkgs/purl" -) - -// distroTypes are PURL types whose namespace names the distribution -// ("alpine", "debian"), not part of the package name the handlers store. -var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} - -// DistroType reports whether a PURL type's namespace names the distribution, -// so that handlers ignore it. -func DistroType(purlType string) bool { - return distroTypes[purlType] -} - -// HandlerEcosystem returns the ecosystem name the proxy's handlers store -// packages of a PURL type under. purl names gem and composer after their -// registries, rubygems and packagist; the handlers use the type. -func HandlerEcosystem(purlType string) string { - switch purlType { - case "gem", "composer": - return purlType - default: - return purl.PURLTypeToEcosystem(purlType) - } -} - -// HandlerPackage returns the ecosystem and package name the proxy's handlers -// store p under, so a package named in a PURL, such as one from an SBOM, -// matches the package a client fetched through the proxy: -// -// - apk, alpm, bsd, deb and rpm: the namespace names the distribution and -// is dropped -// - conda: the channel, from the namespace or the channel qualifier, is -// part of the name ("conda-forge/numpy"); without either the name stays -// bare -// - pypi and nuget: the name in purl's comparison form, PEP 503 for PyPI -// and lowercased for NuGet, as the handlers store them -// - swift: namespace/name, the registry scope or the source repository -// with the package, which the Swift mirror resolves; purl's full name -// is the registry identity scope.name -// -// Other types use purl's full name, which keeps the namespace: "@babel/core", -// or "org.slf4j:slf4j-api" for Maven, as its handler stores it. -func HandlerPackage(p *purl.PURL) (ecosystem, name string) { - ecosystem = HandlerEcosystem(p.Type) - switch { - case distroTypes[p.Type]: - return ecosystem, p.Name - case p.Type == "conda": - return ecosystem, condaName(p.Namespace, p.Name, p.Qualifier("channel")) - case p.Type == "pypi" || p.Type == "nuget": - return ecosystem, p.ComparisonForm().FullName() - case p.Type == "swift" && p.Namespace != "": - return ecosystem, p.Namespace + "/" + p.Name - } - return ecosystem, p.FullName() -} - -func condaName(namespace, name, channel string) string { - switch { - case namespace != "": - return namespace + "/" + name - case channel != "": - return channel + "/" + name - default: - return name - } -} diff --git a/internal/packageurl/handler_test.go b/internal/packageurl/handler_test.go deleted file mode 100644 index 7283b9f..0000000 --- a/internal/packageurl/handler_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package packageurl - -import ( - "testing" - - "github.com/git-pkgs/purl" -) - -// HandlerPackage is used by both the mirror and the project inventory, so a -// mirrored package, a package listed in an SBOM and the same package fetched -// by a client end up under one name. -func TestHandlerPackage(t *testing.T) { - tests := []struct { - purl string - ecosystem string - name string - }{ - // purl names these after their registries; the handlers use the type. - {"pkg:gem/rake@13.2.1", "gem", "rake"}, - {"pkg:composer/psr/log@3.0.2", "composer", "psr/log"}, - // Distribution namespaces are dropped. - {"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64", "alpine", "busybox"}, - {"pkg:apk/curl@8.14.1-r2", "alpine", "curl"}, - {"pkg:alpm/arch/glibc@2.39-1", "arch", "glibc"}, - {"pkg:bsd/freebsd/curl@8.7.1", "freebsd", "curl"}, - {"pkg:deb/debian/libc6@2.36-9?arch=amd64", "deb", "libc6"}, - {"pkg:rpm/fedora/curl@8.6.0-7.fc40", "rpm", "curl"}, - // The conda channel is part of the name. - {"pkg:conda/numpy@1.26.4?channel=conda-forge", "conda", "conda-forge/numpy"}, - {"pkg:conda/conda-forge/numpy@1.26.4", "conda", "conda-forge/numpy"}, - {"pkg:conda/numpy@1.26.4", "conda", "numpy"}, - // PyPI names are normalized as PEP 503 does, NuGet IDs lowercased. - {"pkg:pypi/Zope.Interface@6.4", "pypi", "zope-interface"}, - {"pkg:pypi/zope_interface", "pypi", "zope-interface"}, - {"pkg:nuget/Newtonsoft.Json@13.0.3", "nuget", "newtonsoft.json"}, - // Other namespaces stay part of the name. - {"pkg:npm/%40babel/core@7.23.0", "npm", "@babel/core"}, - // The Maven handler stores "group:artifact". - {"pkg:maven/org.slf4j/slf4j-api@2.0.13", "maven", "org.slf4j:slf4j-api"}, - {"pkg:docker/library/alpine@3.20", "docker", "library/alpine"}, - {"pkg:julia/Example@0.5.3?uuid=7876af07-990d-54b4-ab0e-23690620f79a", "julia", "Example"}, - // Swift keeps namespace/name, the scope or the source repository, - // which the Swift mirror splits again; purl's full name would be the - // registry identity "scope.name". - {"pkg:swift/Acme/Widget@1.0.0", "swift", "Acme/Widget"}, - {"pkg:swift/github.com/apple/swift-nio@2.62.0", "swift", "github.com/apple/swift-nio"}, - // Go module paths keep their case. - {"pkg:golang/github.com/Azure/foo@v1.2.0", "golang", "github.com/Azure/foo"}, - } - for _, tt := range tests { - t.Run(tt.purl, func(t *testing.T) { - p, err := purl.Parse(tt.purl) - if err != nil { - t.Fatal(err) - } - ecosystem, name := HandlerPackage(p) - if ecosystem != tt.ecosystem || name != tt.name { - t.Errorf("HandlerPackage(%q) = %q, %q, want %q, %q", tt.purl, ecosystem, name, tt.ecosystem, tt.name) - } - }) - } -} diff --git a/internal/packageurl/packageurl.go b/internal/packageurl/packageurl.go deleted file mode 100644 index e01f4b2..0000000 --- a/internal/packageurl/packageurl.go +++ /dev/null @@ -1,65 +0,0 @@ -// Package packageurl builds package URLs from ecosystem-native package names. -package packageurl - -import ( - "strings" - - "github.com/git-pkgs/purl" -) - -// Make constructs a package URL from an ecosystem-native package name. -func Make(ecosystem, name, version string) *purl.PURL { - return purl.MakePURL(ecosystem, name, version) -} - -// MakeString constructs a package URL string. It returns an empty string when -// the package identity cannot be represented as a PURL. -func MakeString(ecosystem, name, version string) string { - return purl.MakePURLString(ecosystem, name, version) -} - -// PolicyPackageKey maps a canonical versionless PURL to its policy key. -// Go policies match module names case-insensitively for compatibility. -func PolicyPackageKey(packagePURL string) string { - if strings.HasPrefix(packagePURL, "pkg:golang/") { - return strings.ToLower(packagePURL) - } - return packagePURL -} - -// WithVersionString returns a package PURL with its version replaced. It -// returns an empty string when packagePURL is invalid. -func WithVersionString(packagePURL, version string) string { - pkg, err := purl.Parse(packagePURL) - if err != nil { - return "" - } - return pkg.WithVersion(version).String() -} - -// MakeCacheStrings returns package and version PURLs suitable for artifact -// cache records. Swift registry identities keep their generic storage PURLs. -// Cache keys are independent of the configured upstream. -func MakeCacheStrings(ecosystem, name, version string) (packagePURL, versionPURL string) { - if purl.NormalizeEcosystem(ecosystem) == "swift" { - if identity, ok := swiftRegistryIdentity(name); ok { - pkg := purl.New("generic", "swift-registry", identity, "", nil) - return pkg.String(), pkg.WithVersion(version).String() - } - } - if pkg := Make(ecosystem, name, ""); pkg != nil { - return pkg.String(), pkg.WithVersion(version).String() - } - return "", "" -} - -func swiftRegistryIdentity(name string) (string, bool) { - scope, packageName, found := strings.Cut(name, "/") - if !found { - scope, packageName, found = strings.Cut(name, ".") - } - if !found || scope == "" || packageName == "" || strings.ContainsAny(packageName, "/.") { - return "", false - } - return strings.ToLower(scope) + "." + strings.ToLower(packageName), true -} diff --git a/internal/packageurl/packageurl_test.go b/internal/packageurl/packageurl_test.go deleted file mode 100644 index 5a46776..0000000 --- a/internal/packageurl/packageurl_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package packageurl - -import "testing" - -func TestMakeSwiftRegistryPathUnsupported(t *testing.T) { - if got := Make("swift", "apple/swift-argument-parser", "1.8.2"); got != nil { - t.Errorf("Make() = %q, want nil", got.String()) - } - if got := MakeString("swift", "apple/swift-argument-parser", "1.8.2"); got != "" { - t.Errorf("MakeString() = %q, want empty string", got) - } -} - -func TestMakeStringSwiftSourceCoordinate(t *testing.T) { - got := MakeString("swift", "github.com/apple/swift-package-manager", "1.7.0") - want := "pkg:swift/github.com/apple/swift-package-manager@1.7.0" - if got != want { - t.Errorf("MakeString() = %q, want %q", got, want) - } -} - -func TestWithVersionStringPreservesQualifiers(t *testing.T) { - packagePURL := "pkg:generic/swift-registry/apple.example?repository_url=https:%2F%2Fold.example%2Fswift" - got := WithVersionString(packagePURL, "1.2.3") - want := "pkg:generic/swift-registry/apple.example@1.2.3?repository_url=https:%2F%2Fold.example%2Fswift" - if got != want { - t.Errorf("WithVersionString() = %q, want %q", got, want) - } - - if got := WithVersionString("not a purl", "1.2.3"); got != "" { - t.Errorf("WithVersionString() = %q for invalid PURL, want empty string", got) - } -} - -func TestMakeCacheStringsSwiftRegistryIdentity(t *testing.T) { - packagePURL, versionPURL := MakeCacheStrings("swift", "APPLE/EXAMPLE", "1.2.3") - - wantPackage := "pkg:generic/swift-registry/apple.example" - if packagePURL != wantPackage { - t.Errorf("package PURL = %q, want %q", packagePURL, wantPackage) - } - wantVersion := "pkg:generic/swift-registry/apple.example@1.2.3" - if versionPURL != wantVersion { - t.Errorf("version PURL = %q, want %q", versionPURL, wantVersion) - } - - for _, identity := range []string{"apple.example", "APPLE.EXAMPLE", "apple/example"} { - dottedPackage, dottedVersion := MakeCacheStrings("Swift", identity, "1.2.3") - if dottedPackage != packagePURL || dottedVersion != versionPURL { - t.Errorf("%q cache PURLs = %q, %q; want %q, %q", identity, dottedPackage, dottedVersion, packagePURL, versionPURL) - } - } -} - -func TestMakeCacheStringsUsesSourcePURLWhenAvailable(t *testing.T) { - packagePURL, versionPURL := MakeCacheStrings("swift", "github.com/apple/swift-package-manager", "1.7.0") - - if packagePURL != "pkg:swift/github.com/apple/swift-package-manager" { - t.Errorf("package PURL = %q", packagePURL) - } - if versionPURL != "pkg:swift/github.com/apple/swift-package-manager@1.7.0" { - t.Errorf("version PURL = %q", versionPURL) - } -} - -func TestMakeStringDelegatesOtherEcosystems(t *testing.T) { - got := MakeString("npm", "@babel/core", "7.23.0") - want := "pkg:npm/%40babel/core@7.23.0" - if got != want { - t.Errorf("MakeString() = %q, want %q", got, want) - } -} diff --git a/internal/retention/retention.go b/internal/retention/retention.go deleted file mode 100644 index 156e410..0000000 --- a/internal/retention/retention.go +++ /dev/null @@ -1,272 +0,0 @@ -// Package retention holds the per-ecosystem rules for evicting cached -// artifacts that have not been accessed for a configured time. -// -// An ecosystem takes part only once its handler registers a Spec with -// Default, usually from an init function in the handler's own file. Until -// then configuring retention for it is a validation error, and the global -// default does not apply to it. -package retention - -import ( - "fmt" - "slices" - "sort" - "sync" - "time" - - "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/purl" -) - -// KnownKeys lists the ecosystem keys storage.retention.ecosystems accepts. -// They are the literals the handlers pass when they fetch an artifact. A key -// outside this list is a typo; a key in it whose handler has not registered -// yet is not supported by this version. -var KnownKeys = []string{ - "alpine", "cargo", "composer", "conan", "conda", "cran", "deb", "freebsd", "gem", - "generic", "golang", "helm", "hex", "julia", "maven", "npm", "nuget", "oci", "pub", - "pypi", "rpm", "swift", -} - -// IsKnown reports whether key is one of KnownKeys. -func IsKnown(key string) bool { - return slices.Contains(KnownKeys, key) -} - -// Spec describes how one ecosystem takes part in retention. -type Spec struct { - // Key is the ecosystem's config key, one of KnownKeys. - Key string - - // ExtraDBEcosystems lists packages.ecosystem values besides the ones - // derived from Key that belong to this ecosystem. - ExtraDBEcosystems []string - - // CanonicalPackage turns a package PURL from the config into a - // policy key, reporting false when the - // PURL does not belong to this ecosystem. Nil means the ecosystem accepts - // no package overrides. - CanonicalPackage func(p *purl.PURL) (string, bool) - - // SourcePackage reports a package PURL that names the package by where - // its source lives, such as a Swift package by its repository. It has to - // be resolved through the upstream before it can be compared, so the - // config keeps it in Rules.Sources until then. Nil means the ecosystem - // has no such form. - SourcePackage func(p *purl.PURL) bool -} - -// dbEcosystems returns every packages.ecosystem value that belongs to the -// spec: the key, its PURL type and the ecosystem name that type maps back -// to, the same spellings git-pkgs matches. The proxy's mirror command and -// git-pkgs store "rubygems" where the gem handler stores "gem", for example. -func (s Spec) dbEcosystems() []string { - purlType := purl.EcosystemToPURLType(s.Key) - values := []string{s.Key, purlType, purl.PURLTypeToEcosystem(purlType)} - values = append(values, s.ExtraDBEcosystems...) - slices.Sort(values) - return slices.Compact(values) -} - -// DefaultCanonical returns a CanonicalPackage for ecosystems that store the -// package PURL exactly as purl.MakePURLString builds it from the PURL's full -// name. It accepts a PURL only when building it back that way gives the same -// package, so for an ecosystem that stores a different form, such as one -// with a default namespace, it refuses the key instead of producing one -// that never matches. Such ecosystems need their own CanonicalPackage. -func DefaultCanonical(key string) func(p *purl.PURL) (string, bool) { - purlType := purl.EcosystemToPURLType(key) - return func(p *purl.PURL) (string, bool) { - if p == nil || p.Type != purlType { - return "", false - } - canonical := purl.MakePURLString(key, p.FullName(), "") - if canonical == "" { - return "", false - } - given := purl.New(p.Type, p.Namespace, p.Name, "", nil).String() - return canonical, canonical == given - } -} - -// Registry maps ecosystem keys to their specs. -type Registry struct { - mu sync.RWMutex - specs map[string]Spec - byDB map[string]string -} - -// NewRegistry returns an empty registry. -func NewRegistry() *Registry { - return &Registry{specs: map[string]Spec{}, byDB: map[string]string{}} -} - -// Default is the registry the handlers register with. -var Default = NewRegistry() - -// Register adds s. It panics when the key is not one of KnownKeys, is -// already registered, or claims a packages.ecosystem value another spec -// claims, since each of those is a programming error. -func (r *Registry) Register(s Spec) { - r.mu.Lock() - defer r.mu.Unlock() - - if !IsKnown(s.Key) { - panic(fmt.Sprintf("retention: unknown ecosystem key %q", s.Key)) - } - if _, ok := r.specs[s.Key]; ok { - panic(fmt.Sprintf("retention: ecosystem %q registered twice", s.Key)) - } - values := s.dbEcosystems() - for _, v := range values { - if owner, ok := r.byDB[v]; ok { - panic(fmt.Sprintf("retention: ecosystem value %q claimed by both %q and %q", v, owner, s.Key)) - } - } - r.specs[s.Key] = s - for _, v := range values { - r.byDB[v] = s.Key - } -} - -// Lookup returns the spec registered for key. -func (r *Registry) Lookup(key string) (Spec, bool) { - r.mu.RLock() - defer r.mu.RUnlock() - s, ok := r.specs[key] - return s, ok -} - -// Keys returns the registered keys in sorted order. -func (r *Registry) Keys() []string { - r.mu.RLock() - defer r.mu.RUnlock() - keys := make([]string, 0, len(r.specs)) - for k := range r.specs { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - -// KeyForDBEcosystem returns the registered key a packages.ecosystem value -// belongs to. -func (r *Registry) KeyForDBEcosystem(ecosystem string) (string, bool) { - r.mu.RLock() - defer r.mu.RUnlock() - key, ok := r.byDB[ecosystem] - return key, ok -} - -// Canonical returns the policy key for a configured package PURL. -func (r *Registry) Canonical(p *purl.PURL) (string, bool) { - for _, key := range r.Keys() { - s, _ := r.Lookup(key) - if s.CanonicalPackage == nil { - continue - } - if canonical, ok := s.CanonicalPackage(p); ok { - return canonical, true - } - } - return "", false -} - -// Source returns the ecosystem key whose SourcePackage claims p. -func (r *Registry) Source(p *purl.PURL) (string, bool) { - for _, key := range r.Keys() { - s, _ := r.Lookup(key) - if s.SourcePackage != nil && s.SourcePackage(p) { - return key, true - } - } - return "", false -} - -// Rules are resolved retention durations. A duration of zero means an -// artifact is never evicted by age. -type Rules struct { - // Default applies to registered ecosystems without their own rule, - // except generic, which requires an explicit rule. - Default time.Duration - // Ecosystems holds rules keyed by registered ecosystem key. - Ecosystems map[string]time.Duration - // Packages holds rules keyed by package PURL, with Go keys lowercased. - Packages map[string]time.Duration - // Sources holds package rules not resolved to a stored package PURL - // yet, by ecosystem key and then by the configured PURL (see - // Spec.SourcePackage). An ecosystem with any is not swept, since one of - // them may keep an artifact the sweep would otherwise clear. - Sources map[string]map[string]time.Duration -} - -// Pending reports whether the ecosystem key has package rules still to be -// resolved, and must not be swept. -func (r Rules) Pending(key string) bool { - return len(r.Sources[key]) > 0 -} - -// Resolve moves the rule for source in ecosystem key to every stored package -// PURL it resolved to. A different rule already set for any of them is a -// conflict, and the source stays pending with none of them set. -func (r Rules) Resolve(key, source string, canonical []string) error { - d, ok := r.Sources[key][source] - if !ok { - return nil - } - if len(canonical) == 0 { - return fmt.Errorf("%s resolves to no stored package", source) - } - for _, c := range canonical { - if prev, set := r.Packages[c]; set && prev != d { - return fmt.Errorf("%s resolves to %s, which another rule sets differently", source, c) - } - } - for _, c := range canonical { - r.Packages[c] = d - } - delete(r.Sources[key], source) - return nil -} - -// For returns the retention for an artifact of the given ecosystem key and -// stored package PURL: the package rule, else the ecosystem rule, else the -// default. Generic artifacts require an explicit package or ecosystem rule. -func (r Rules) For(key, packagePURL string) time.Duration { - packagePURL = packageurl.PolicyPackageKey(packagePURL) - if d, ok := r.Packages[packagePURL]; ok { - return d - } - if d, ok := r.Ecosystems[key]; ok { - return d - } - // Release assets may disappear upstream, so keep generic opt-in. - if key == "generic" { - return 0 - } - return r.Default -} - -// MinPositive returns the shortest non-zero duration among all rules, or -// zero when every rule is zero. -func (r Rules) MinPositive() time.Duration { - minimum := time.Duration(0) - consider := func(d time.Duration) { - if d > 0 && (minimum == 0 || d < minimum) { - minimum = d - } - } - consider(r.Default) - for _, d := range r.Ecosystems { - consider(d) - } - for _, d := range r.Packages { - consider(d) - } - for _, sources := range r.Sources { - for _, d := range sources { - consider(d) - } - } - return minimum -} diff --git a/internal/retention/retention_test.go b/internal/retention/retention_test.go deleted file mode 100644 index 5036375..0000000 --- a/internal/retention/retention_test.go +++ /dev/null @@ -1,244 +0,0 @@ -package retention - -import ( - "slices" - "testing" - "time" - - "github.com/git-pkgs/purl" -) - -func TestSpecDBEcosystems(t *testing.T) { - tests := []struct { - key string - want []string - }{ - {"npm", []string{"npm"}}, - {"gem", []string{"gem", "rubygems"}}, - {"composer", []string{"composer", "packagist"}}, - {"alpine", []string{"alpine", "apk"}}, - {"golang", []string{"golang"}}, - {"deb", []string{"deb"}}, - } - for _, tt := range tests { - t.Run(tt.key, func(t *testing.T) { - got := Spec{Key: tt.key}.dbEcosystems() - if !slices.Equal(got, tt.want) { - t.Errorf("dbEcosystems() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestRegistryRegister(t *testing.T) { - reg := NewRegistry() - reg.Register(Spec{Key: "gem"}) - reg.Register(Spec{Key: "npm"}) - - if got := reg.Keys(); !slices.Equal(got, []string{"gem", "npm"}) { - t.Errorf("Keys() = %v, want [gem npm]", got) - } - for _, value := range []string{"gem", "rubygems"} { - if key, ok := reg.KeyForDBEcosystem(value); !ok || key != "gem" { - t.Errorf("KeyForDBEcosystem(%q) = %q, %v, want gem, true", value, key, ok) - } - } - if _, ok := reg.KeyForDBEcosystem("pypi"); ok { - t.Error("KeyForDBEcosystem(pypi) found a key for an unregistered ecosystem") - } - if _, ok := reg.Lookup("pypi"); ok { - t.Error("Lookup(pypi) found an unregistered ecosystem") - } -} - -func TestRegistryRegisterPanics(t *testing.T) { - tests := []struct { - name string - specs []Spec - }{ - {"unknown key", []Spec{{Key: "not-an-ecosystem"}}}, - {"duplicate key", []Spec{{Key: "npm"}, {Key: "npm"}}}, - {"overlapping ecosystem value", []Spec{{Key: "gem"}, {Key: "npm", ExtraDBEcosystems: []string{"rubygems"}}}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - reg := NewRegistry() - defer func() { - if recover() == nil { - t.Error("Register did not panic") - } - }() - for _, s := range tt.specs { - reg.Register(s) - } - }) - } -} - -func mustParse(t *testing.T, s string) *purl.PURL { - t.Helper() - p, err := purl.Parse(s) - if err != nil { - t.Fatalf("parsing %q: %v", s, err) - } - return p -} - -func TestDefaultCanonical(t *testing.T) { - canonical := DefaultCanonical("npm") - - got, ok := canonical(mustParse(t, "pkg:npm/%40babel/core")) - if !ok || got != "pkg:npm/%40babel/core" { - t.Errorf("scoped package = %q, %v, want pkg:npm/%%40babel/core, true", got, ok) - } - if got, ok := canonical(mustParse(t, "pkg:pypi/requests")); ok { - t.Errorf("other type accepted as %q", got) - } - if _, ok := canonical(nil); ok { - t.Error("nil PURL accepted") - } - - // alpine stores pkg:apk/alpine/{name} and deb stores pkg:deb/{name}, so - // building their keys back from the full name gives a different package. - // The default refuses them rather than produce keys that never match. - for _, tt := range []struct{ key, purl string }{ - {"alpine", "pkg:apk/alpine/curl"}, - {"alpine", "pkg:apk/curl"}, - {"deb", "pkg:deb/debian/curl"}, - } { - if got, ok := DefaultCanonical(tt.key)(mustParse(t, tt.purl)); ok { - t.Errorf("DefaultCanonical(%q) accepted %s as %q", tt.key, tt.purl, got) - } - } -} - -func TestRegistryCanonical(t *testing.T) { - reg := NewRegistry() - reg.Register(Spec{Key: "julia"}) - reg.Register(Spec{Key: "npm", CanonicalPackage: DefaultCanonical("npm")}) - - if got, ok := reg.Canonical(mustParse(t, "pkg:npm/lodash")); !ok || got != "pkg:npm/lodash" { - t.Errorf("Canonical(npm) = %q, %v, want pkg:npm/lodash, true", got, ok) - } - if got, ok := reg.Canonical(mustParse(t, "pkg:cargo/serde")); ok { - t.Errorf("Canonical(cargo) = %q for an unregistered ecosystem", got) - } -} - -func TestRulesFor(t *testing.T) { - rules := Rules{ - Default: 30 * 24 * time.Hour, - Ecosystems: map[string]time.Duration{"npm": 14 * 24 * time.Hour, "maven": 0}, - Packages: map[string]time.Duration{"pkg:npm/lodash": 0, "pkg:maven/a/b": 90 * 24 * time.Hour}, - } - tests := []struct { - key, pkg string - want time.Duration - }{ - {"npm", "pkg:npm/lodash", 0}, - {"npm", "pkg:npm/left-pad", 14 * 24 * time.Hour}, - {"maven", "pkg:maven/a/b", 90 * 24 * time.Hour}, - {"maven", "pkg:maven/c/d", 0}, - {"cargo", "pkg:cargo/serde", 30 * 24 * time.Hour}, - } - for _, tt := range tests { - if got := rules.For(tt.key, tt.pkg); got != tt.want { - t.Errorf("For(%q, %q) = %v, want %v", tt.key, tt.pkg, got, tt.want) - } - } -} - -func TestGenericRulesRequireExplicitConfiguration(t *testing.T) { - day := 24 * time.Hour - const pkg = "pkg:generic/jqlang%2Fjq" - tests := []struct { - name string - ecosystems map[string]time.Duration - packages map[string]time.Duration - want time.Duration - }{ - {"global default ignored", nil, nil, 0}, - {"ecosystem rule", map[string]time.Duration{"generic": 7 * day}, nil, 7 * day}, - {"ecosystem disabled", map[string]time.Duration{"generic": 0}, nil, 0}, - {"package only", nil, map[string]time.Duration{pkg: 14 * day}, 14 * day}, - {"other package ignored", nil, map[string]time.Duration{"pkg:generic/other%2Frepo": day}, 0}, - {"package overrides ecosystem", map[string]time.Duration{"generic": 7 * day}, map[string]time.Duration{pkg: 14 * day}, 14 * day}, - {"package disables eviction", map[string]time.Duration{"generic": 7 * day}, map[string]time.Duration{pkg: 0}, 0}, - {"package enables eviction", map[string]time.Duration{"generic": 0}, map[string]time.Duration{pkg: day}, day}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - rules := Rules{Default: 30 * day, Ecosystems: tt.ecosystems, Packages: tt.packages} - if got := rules.For("generic", pkg); got != tt.want { - t.Errorf("generic retention = %v, want %v", got, tt.want) - } - if got := rules.For("npm", "pkg:npm/lodash"); got != 30*day { - t.Errorf("npm default retention = %v, want 30d", got) - } - }) - } -} - -func TestRulesMinPositive(t *testing.T) { - if got := (Rules{}).MinPositive(); got != 0 { - t.Errorf("empty rules MinPositive() = %v, want 0", got) - } - rules := Rules{ - Default: 0, - Ecosystems: map[string]time.Duration{"npm": 14 * time.Hour, "maven": 0}, - Packages: map[string]time.Duration{"pkg:npm/lodash": 2 * time.Hour}, - } - if got := rules.MinPositive(); got != 2*time.Hour { - t.Errorf("MinPositive() = %v, want 2h", got) - } -} - -func TestRulesResolveSources(t *testing.T) { - rules := Rules{ - Packages: map[string]time.Duration{"pkg:x/kept": 0}, - Sources: map[string]map[string]time.Duration{"swift": { - "pkg:swift/github.com/a/one": 0, - "pkg:swift/github.com/a/two": time.Hour, - }}, - } - if got := rules.MinPositive(); got != time.Hour { - t.Errorf("MinPositive = %v, want the pending rule's hour", got) - } - if err := rules.Resolve("swift", "pkg:swift/github.com/a/two", []string{"pkg:x/kept"}); err == nil { - t.Error("a rule resolving to a package another rule sets differently was accepted") - } - if !rules.Pending("swift") { - t.Fatal("conflicting rule no longer pending") - } - if err := rules.Resolve("swift", "pkg:swift/github.com/a/one", []string{"pkg:x/kept"}); err != nil { - t.Errorf("same duration for the same package: %v", err) - } - if err := rules.Resolve("swift", "pkg:swift/github.com/a/two", []string{"pkg:x/two"}); err != nil { - t.Fatal(err) - } - if rules.Pending("swift") || rules.Packages["pkg:x/two"] != time.Hour { - t.Errorf("after resolving: sources %v, packages %v", rules.Sources, rules.Packages) - } -} - -// A source that resolves to several stored packages sets the rule for all -// of them, or, on a conflict with any, for none. -func TestRulesResolveSeveralPackages(t *testing.T) { - rules := Rules{ - Packages: map[string]time.Duration{"pkg:x/b": time.Hour}, - Sources: map[string]map[string]time.Duration{"swift": {"pkg:swift/github.com/a/one": 0}}, - } - if err := rules.Resolve("swift", "pkg:swift/github.com/a/one", []string{"pkg:x/a", "pkg:x/b"}); err == nil { - t.Error("a rule conflicting for one of its packages was accepted") - } - if _, set := rules.Packages["pkg:x/a"]; set || !rules.Pending("swift") { - t.Fatalf("conflicting rule partly applied: sources %v, packages %v", rules.Sources, rules.Packages) - } - delete(rules.Packages, "pkg:x/b") - if err := rules.Resolve("swift", "pkg:swift/github.com/a/one", []string{"pkg:x/a", "pkg:x/b"}); err != nil { - t.Fatal(err) - } - if rules.Pending("swift") || len(rules.Packages) != 2 { - t.Errorf("after resolving: sources %v, packages %v", rules.Sources, rules.Packages) - } -} diff --git a/internal/scanner/group.go b/internal/scanner/group.go deleted file mode 100644 index a4856e0..0000000 --- a/internal/scanner/group.go +++ /dev/null @@ -1,252 +0,0 @@ -package scanner - -import ( - "context" - "errors" - "fmt" - "log/slog" - "net/http" - "sync" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/metrics" -) - -const ( - modeBlock = "block" - modeMonitor = "monitor" - - defaultTimeout = 30 * time.Second -) - -type entry struct { - scanner Scanner - mode string - ecosystems map[string]struct{} // nil/empty means all ecosystems -} - -// Group runs a set of configured Scanners concurrently and turns their -// individual verdicts into a single decision. -type Group struct { - entries []entry - timeout time.Duration - failOpen bool - logger *slog.Logger -} - -// NewGroup builds a Group from cfg. If cfg.Enabled is false, the returned -// Group has no entries and Enabled() reports false, so callers can skip -// the scan path entirely. -func NewGroup(cfg config.ScanningConfig, logger *slog.Logger) (*Group, error) { - g := &Group{ - timeout: defaultTimeout, - failOpen: cfg.FailOpen, - logger: logger, - } - if !cfg.Enabled { - return g, nil - } - if cfg.SigningKeyExpanded() == "" { - return nil, fmt.Errorf("scanning.signing_key is required when scanning.enabled is true") - } - if d, err := time.ParseDuration(cfg.Timeout); err == nil && d > 0 { - g.timeout = d - } - - for _, sc := range cfg.Scanners { - mode := sc.Mode - if mode == "" { - mode = modeBlock - } - if mode != modeBlock && mode != modeMonitor { - return nil, fmt.Errorf("scanner %q: invalid mode %q (must be %q or %q)", sc.Name, mode, modeBlock, modeMonitor) - } - - var ecosystems map[string]struct{} - if len(sc.Ecosystems) > 0 { - ecosystems = make(map[string]struct{}, len(sc.Ecosystems)) - for _, eco := range sc.Ecosystems { - key, err := config.ScannerEcosystemKey(eco) - if err != nil { - return nil, fmt.Errorf("scanner %q: %w", sc.Name, err) - } - ecosystems[key] = struct{}{} - } - } - - g.entries = append(g.entries, entry{ - scanner: NewHTTPScanner(sc.Name, sc.URL, sc.HeadersExpanded(), http.DefaultClient), - mode: mode, - ecosystems: ecosystems, - }) - } - - return g, nil -} - -// Enabled reports whether any scanner is configured. -func (g *Group) Enabled() bool { - return g != nil && len(g.entries) > 0 -} - -// Covers reports whether any scanner, in either mode, applies to ecosystem. -func (g *Group) Covers(ecosystem string) bool { - return g.Enabled() && len(g.applicable(ecosystem)) > 0 -} - -// Blocks reports whether a block-mode scanner applies to ecosystem, so an -// artifact must be scanned whole before any of it is served. -func (g *Group) Blocks(ecosystem string) bool { - if !g.Enabled() { - return false - } - for _, e := range g.applicable(ecosystem) { - if e.mode == modeBlock { - return true - } - } - return false -} - -// Timeout returns the per-scan-call timeout used to bound the signed fetch -// URL's validity. -func (g *Group) Timeout() time.Duration { - return g.timeout -} - -func (g *Group) applicable(ecosystem string) []entry { - // Callers may pass an alias such as go from a PURL type. - ecosystem = config.NormalizeScannerEcosystem(ecosystem) - var out []entry - for _, e := range g.entries { - if len(e.ecosystems) == 0 { - out = append(out, e) - continue - } - if _, ok := e.ecosystems[ecosystem]; ok { - out = append(out, e) - } - } - return out -} - -// Scan runs every scanner applicable to req.Ecosystem concurrently, never -// sequentially, and returns a single decision. -// -// The moment any "block" mode scanner reports Allowed: false (or errors, -// unless FailOpen is set), Scan cancels a context shared by every -// goroutine: in-flight calls to the other scanners are aborted rather than -// waited out, since a single block already decides the outcome. Scan still -// waits for all goroutines to observe that cancellation and return before -// it itself returns, so no scan call outlives this method call. -// -// If nothing blocks, Scan waits for every "block" mode scanner to finish -// before reporting Allowed: true — an allow decision can't be finalized -// until all of them have answered. "monitor" mode scanners never gate the -// wait or trigger cancellation: a monitor verdict of Allowed: false is -// logged and folded into Result.Findings, but never blocks. -func (g *Group) Scan(ctx context.Context, req Request) Result { - entries := g.applicable(req.Ecosystem) - if len(entries) == 0 { - return Result{Allowed: true} - } - - scanCtx, cancel := context.WithTimeout(ctx, g.timeout) - defer cancel() - - var ( - mu sync.Mutex - findings []Finding - blocked *Result - ) - - var wg sync.WaitGroup - for _, e := range entries { - wg.Add(1) - go func(e entry) { - defer wg.Done() - entryFindings, entryBlock := g.evaluate(scanCtx, req, e) - - mu.Lock() - findings = append(findings, entryFindings...) - if entryBlock != nil && blocked == nil { - blocked = entryBlock - cancel() - } - mu.Unlock() - }(e) - } - - wg.Wait() - - if blocked != nil { - blocked.Findings = findings - return *blocked - } - - return Result{Allowed: true, Findings: findings} -} - -// evaluate runs a single scanner and reports its findings plus, if this -// scanner's verdict should block the artifact, the Result to block with -// (nil otherwise). It never sets Result.Findings on a returned block -// Result — the caller assembles Findings from every entry once all of them -// have finished. -func (g *Group) evaluate(scanCtx context.Context, req Request, e entry) (findings []Finding, block *Result) { - start := time.Now() - res, err := e.scanner.Scan(scanCtx, req) - duration := time.Since(start) - - if err != nil { - errType := "error" - switch { - case errors.Is(scanCtx.Err(), context.DeadlineExceeded): - errType = "timeout" - case errors.Is(scanCtx.Err(), context.Canceled): - // scanCtx was cancelled because another scanner in the group - // already decided the verdict, not because this call itself - // timed out or failed on its own. - errType = "cancelled" - } - metrics.RecordScanError(req.Ecosystem, e.scanner.Name(), errType, metrics.FailureDetail{ - PURL: req.PURL, Filename: req.Filename, Err: err}) - - if errType == "cancelled" { - // Another scanner already decided the verdict and cancelled - // scanCtx; this call didn't fail on its own, so don't log it - // as if it did. - return nil, nil - } - - if e.mode == modeMonitor || g.failOpen { - g.logger.Warn("scanner call failed, treating as allowed", - "scanner", e.scanner.Name(), "mode", e.mode, "error", err) - return nil, nil - } - - g.logger.Warn("scanner call failed, blocking artifact", - "scanner", e.scanner.Name(), "mode", e.mode, "error", err) - return nil, &Result{ - Allowed: false, - Reason: fmt.Sprintf("scanner %q failed: %v", e.scanner.Name(), err), - ScannerName: e.scanner.Name(), - InfraError: true, - } - } - - metrics.RecordScanResult(req.Ecosystem, e.scanner.Name(), res.Allowed, duration) - - if e.mode == modeMonitor { - if !res.Allowed { - g.logger.Warn("monitor scanner flagged artifact", - "scanner", e.scanner.Name(), "reason", res.Reason) - } - return res.Findings, nil - } - - if !res.Allowed { - return res.Findings, &Result{Allowed: false, Reason: res.Reason, ScannerName: e.scanner.Name()} - } - return res.Findings, nil -} diff --git a/internal/scanner/group_test.go b/internal/scanner/group_test.go deleted file mode 100644 index 5dca201..0000000 --- a/internal/scanner/group_test.go +++ /dev/null @@ -1,327 +0,0 @@ -package scanner - -import ( - "context" - "errors" - "io" - "log/slog" - "strings" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/metrics" - "github.com/prometheus/client_golang/prometheus/testutil" -) - -func discardLogger() *slog.Logger { - return slog.New(slog.NewTextHandler(io.Discard, nil)) -} - -// fakeScanner is a Scanner test double that can simulate a delay, a fixed -// result or error, and report whether its context was cancelled before it -// returned. -type fakeScanner struct { - name string - delay time.Duration - result Result - err error - cancelled *bool -} - -func (f *fakeScanner) Name() string { return f.name } - -func (f *fakeScanner) Scan(ctx context.Context, _ Request) (Result, error) { - select { - case <-time.After(f.delay): - case <-ctx.Done(): - if f.cancelled != nil { - *f.cancelled = true - } - return Result{}, ctx.Err() - } - return f.result, f.err -} - -func newTestGroup(entries []entry, failOpen bool) *Group { - return &Group{ - entries: entries, - timeout: time.Second, - failOpen: failOpen, - logger: discardLogger(), - } -} - -func TestGroup_Enabled(t *testing.T) { - disabled, err := NewGroup(config.ScanningConfig{Enabled: false}, discardLogger()) - if err != nil { - t.Fatalf("NewGroup() error: %v", err) - } - if disabled.Enabled() { - t.Error("Enabled() = true for disabled config, want false") - } - - enabled, err := NewGroup(config.ScanningConfig{ - Enabled: true, - Timeout: "10s", - SigningKey: "test-signing-key", - Scanners: []config.ScannerConfig{ - {Name: "clamav", URL: "http://clamav.invalid", Mode: "block"}, - }, - }, discardLogger()) - if err != nil { - t.Fatalf("NewGroup() error: %v", err) - } - if !enabled.Enabled() { - t.Error("Enabled() = false for configured scanner, want true") - } -} - -func TestGroup_Covers(t *testing.T) { - var nilGroup *Group - if nilGroup.Covers("npm") { - t.Error("nil group Covers(npm) = true, want false") - } - g, err := NewGroup(config.ScanningConfig{ - Enabled: true, - SigningKey: "test-signing-key", - Scanners: []config.ScannerConfig{ - {Name: "trivy", URL: "http://trivy.invalid", Mode: "monitor", Ecosystems: []string{"npm"}}, - {Name: "clamav", URL: "http://clamav.invalid", Ecosystems: []string{"pypi"}}, - }, - }, discardLogger()) - if err != nil { - t.Fatalf("NewGroup() error: %v", err) - } - for ecosystem, want := range map[string]bool{"npm": true, "pypi": true, "freebsd": false} { - if got := g.Covers(ecosystem); got != want { - t.Errorf("Covers(%s) = %v, want %v", ecosystem, got, want) - } - } - all := newTestGroup([]entry{{mode: modeBlock}}, false) - if !all.Covers("freebsd") { - t.Error("unrestricted scanner: Covers(freebsd) = false, want true") - } -} - -func TestGroup_NewGroup_InvalidMode(t *testing.T) { - _, err := NewGroup(config.ScanningConfig{ - Enabled: true, - SigningKey: "test-signing-key", - Scanners: []config.ScannerConfig{ - {Name: "bad", URL: "http://bad.invalid", Mode: "quarantine"}, - }, - }, discardLogger()) - if err == nil { - t.Fatal("NewGroup() error = nil, want error for invalid mode") - } -} - -func TestGroup_NewGroup_MissingSigningKey(t *testing.T) { - _, err := NewGroup(config.ScanningConfig{ - Enabled: true, - Scanners: []config.ScannerConfig{ - {Name: "clamav", URL: "http://clamav.invalid", Mode: "block"}, - }, - }, discardLogger()) - if err == nil { - t.Fatal("NewGroup() error = nil, want error for missing signing key") - } -} - -func TestGroup_Scan_NoApplicableScanners(t *testing.T) { - g := newTestGroup([]entry{ - { - scanner: &fakeScanner{name: "npm-only", result: Result{Allowed: false}}, - mode: modeBlock, - ecosystems: map[string]struct{}{"npm": {}}, - }, - }, false) - - result := g.Scan(context.Background(), Request{Ecosystem: "pypi"}) - if !result.Allowed { - t.Error("Allowed = false, want true when no scanner applies to the ecosystem") - } -} - -func TestGroup_Scan_Allowed(t *testing.T) { - g := newTestGroup([]entry{ - {scanner: &fakeScanner{name: "clamav", result: Result{Allowed: true}}, mode: modeBlock}, - }, false) - - result := g.Scan(context.Background(), Request{Ecosystem: "npm"}) - if !result.Allowed { - t.Error("Allowed = false, want true") - } -} - -func TestGroup_Scan_Blocked(t *testing.T) { - g := newTestGroup([]entry{ - {scanner: &fakeScanner{name: "clamav", result: Result{Allowed: false, Reason: "malware"}}, mode: modeBlock}, - }, false) - - result := g.Scan(context.Background(), Request{Ecosystem: "npm"}) - if result.Allowed { - t.Error("Allowed = true, want false") - } - if result.Reason != "malware" { - t.Errorf("Reason = %q, want %q", result.Reason, "malware") - } - if result.ScannerName != "clamav" { - t.Errorf("ScannerName = %q, want %q", result.ScannerName, "clamav") - } - if result.InfraError { - t.Error("InfraError = true, want false — this is a genuine scanner verdict, not an infrastructure failure") - } -} - -func TestGroup_Scan_MonitorNeverBlocks(t *testing.T) { - g := newTestGroup([]entry{ - { - scanner: &fakeScanner{name: "trivy", result: Result{ - Allowed: false, - Reason: "cve found", - Findings: []Finding{{Severity: "medium", Title: "CVE-1234"}}, - }}, - mode: modeMonitor, - }, - }, false) - - result := g.Scan(context.Background(), Request{Ecosystem: "npm"}) - if !result.Allowed { - t.Error("Allowed = false, want true — monitor mode must never block") - } - if len(result.Findings) != 1 || result.Findings[0].Title != "CVE-1234" { - t.Errorf("Findings = %+v, want the monitor scanner's finding folded in", result.Findings) - } -} - -func TestGroup_Scan_FirstBlockCancelsOthers(t *testing.T) { - var slowSawCancel bool - g := newTestGroup([]entry{ - {scanner: &fakeScanner{name: "fast-block", result: Result{Allowed: false, Reason: "blocked"}}, mode: modeBlock}, - {scanner: &fakeScanner{name: "slow", delay: 2 * time.Second, cancelled: &slowSawCancel}, mode: modeBlock}, - }, false) - - start := time.Now() - result := g.Scan(context.Background(), Request{Ecosystem: "npm"}) - elapsed := time.Since(start) - - if result.Allowed { - t.Error("Allowed = true, want false") - } - if elapsed >= 2*time.Second { - t.Errorf("Scan() took %v, want it to return promptly once the slow scanner's context was cancelled", elapsed) - } - if !slowSawCancel { - t.Error("slow scanner never observed context cancellation") - } - - if got := testutil.ToFloat64(metrics.ScanErrors.WithLabelValues("npm", "slow", "cancelled")); got != 1 { - t.Errorf("scan_errors{error_type=cancelled} = %v, want 1 — the slow scanner was aborted by a sibling's block, not by its own timeout", got) - } - if got := testutil.ToFloat64(metrics.ScanErrors.WithLabelValues("npm", "slow", "timeout")); got != 0 { - t.Errorf("scan_errors{error_type=timeout} = %v, want 0 — intra-group cancellation must not be mislabelled as a timeout", got) - } -} - -func TestGroup_Scan_WaitsForAllBlockScannersBeforeAllowing(t *testing.T) { - g := newTestGroup([]entry{ - {scanner: &fakeScanner{name: "fast", result: Result{Allowed: true}}, mode: modeBlock}, - {scanner: &fakeScanner{name: "slow", delay: 30 * time.Millisecond, result: Result{Allowed: true}}, mode: modeBlock}, - }, false) - - start := time.Now() - result := g.Scan(context.Background(), Request{Ecosystem: "npm"}) - elapsed := time.Since(start) - - if !result.Allowed { - t.Error("Allowed = false, want true") - } - if elapsed < 30*time.Millisecond { - t.Errorf("Scan() returned after %v, want it to wait for the slower block scanner", elapsed) - } -} - -func TestGroup_Scan_ErrorFailClosedByDefault(t *testing.T) { - g := newTestGroup([]entry{ - {scanner: &fakeScanner{name: "flaky", err: errors.New("connection refused")}, mode: modeBlock}, - }, false) - - result := g.Scan(context.Background(), Request{Ecosystem: "npm"}) - if result.Allowed { - t.Error("Allowed = true, want false — default posture is fail-closed on scanner error") - } - if !result.InfraError { - t.Error("InfraError = false, want true — the block came from a scanner call failure, not a verdict") - } - if !strings.Contains(result.Reason, "connection refused") { - t.Errorf("Reason = %q, want it to include the underlying error for server-side logging", result.Reason) - } -} - -func TestGroup_Scan_ErrorFailOpen(t *testing.T) { - g := newTestGroup([]entry{ - {scanner: &fakeScanner{name: "flaky", err: errors.New("connection refused")}, mode: modeBlock}, - }, true) - - result := g.Scan(context.Background(), Request{Ecosystem: "npm"}) - if !result.Allowed { - t.Error("Allowed = false, want true — FailOpen must treat scanner errors as allowed") - } -} - -func TestGroupBlocks(t *testing.T) { - g, err := NewGroup(config.ScanningConfig{ - Enabled: true, - SigningKey: "key", - Scanners: []config.ScannerConfig{ - {Name: "watch", URL: "http://watch.invalid", Mode: "monitor"}, - {Name: "gate", URL: "http://gate.invalid", Mode: "block", Ecosystems: []string{"npm"}}, - }, - }, slog.New(slog.NewTextHandler(io.Discard, nil))) - if err != nil { - t.Fatal(err) - } - if !g.Blocks("npm") { - t.Error("Blocks(npm) = false with a block-mode scanner for npm") - } - if g.Blocks("pypi") { - t.Error("Blocks(pypi) = true with only a monitor-mode scanner for it") - } - var none *Group - if none.Blocks("npm") { - t.Error("nil group blocks") - } -} - -func TestGroup_NormalisesEcosystemAliases(t *testing.T) { - g, err := NewGroup(config.ScanningConfig{ - Enabled: true, - SigningKey: "test-signing-key", - Scanners: []config.ScannerConfig{ - {Name: "clamav", URL: "http://clamav.invalid", Ecosystems: []string{"go", "docker", "rubygems"}}, - }, - }, discardLogger()) - if err != nil { - t.Fatalf("NewGroup() error: %v", err) - } - for _, ecosystem := range []string{"golang", "oci", "gem", "go", "Docker", "rubygems"} { - if !g.Blocks(ecosystem) { - t.Errorf("Blocks(%s) = false, want true", ecosystem) - } - } -} - -func TestNewGroup_RejectsUnknownEcosystem(t *testing.T) { - _, err := NewGroup(config.ScanningConfig{ - Enabled: true, - SigningKey: "test-signing-key", - Scanners: []config.ScannerConfig{ - {Name: "clamav", URL: "http://clamav.invalid", Ecosystems: []string{"npm", "pypy"}}, - }, - }, discardLogger()) - if err == nil || !strings.Contains(err.Error(), "pypy") { - t.Errorf("NewGroup() error = %v, want unknown ecosystem pypy", err) - } -} diff --git a/internal/scanner/http.go b/internal/scanner/http.go deleted file mode 100644 index 90a4e5e..0000000 --- a/internal/scanner/http.go +++ /dev/null @@ -1,97 +0,0 @@ -package scanner - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" -) - -// HTTPScanner adapts an external HTTP scanning service to the Scanner -// interface. It POSTs a small JSON notification describing the staged -// artifact, including a signed fetch URL; the external service is -// responsible for GETting that URL itself, running the real scan against -// those bytes, and replying with a verdict before the request's deadline. -// -// Request body: -// -// { -// "ecosystem": "npm", "name": "left-pad", "version": "1.0.0", -// "filename": "left-pad-1.0.0.tgz", "purl": "pkg:npm/left-pad@1.0.0", -// "content_type": "application/octet-stream", "size": 1234, -// "fetch_url": "https://proxy.internal/_internal/scan-fetch?..." -// } -// -// Response body: -// -// { -// "allowed": true, "reason": "", -// "findings": [{"severity": "high", "title": "...", "description": "..."}] -// } -// -// Any compliant adapter — a trivy wrapper, a clamav-rest bridge, a Wiz -// connector, or an in-house service — need only implement this contract. -type HTTPScanner struct { - name string - url string - headers map[string]string - client *http.Client -} - -// NewHTTPScanner creates an HTTPScanner named name that notifies url of -// staged artifacts, attaching headers to every request (e.g. for auth). -// If client is nil, http.DefaultClient is used. -func NewHTTPScanner(name, url string, headers map[string]string, client *http.Client) *HTTPScanner { - if client == nil { - client = http.DefaultClient - } - return &HTTPScanner{name: name, url: url, headers: headers, client: client} -} - -// Name returns the scanner's configured name. -func (s *HTTPScanner) Name() string { return s.name } - -type httpScanResponse struct { - Allowed bool `json:"allowed"` - Reason string `json:"reason"` - Findings []Finding `json:"findings"` -} - -// Scan notifies the configured URL of req and waits for a verdict. -func (s *HTTPScanner) Scan(ctx context.Context, req Request) (Result, error) { - body, err := json.Marshal(req) - if err != nil { - return Result{}, fmt.Errorf("marshal scan request: %w", err) - } - - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, s.url, bytes.NewReader(body)) - if err != nil { - return Result{}, fmt.Errorf("build scan request: %w", err) - } - httpReq.Header.Set("Content-Type", "application/json") - for k, v := range s.headers { - httpReq.Header.Set(k, v) - } - - resp, err := s.client.Do(httpReq) - if err != nil { - return Result{}, fmt.Errorf("calling scanner %q: %w", s.name, err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - return Result{}, fmt.Errorf("scanner %q returned status %d", s.name, resp.StatusCode) - } - - var out httpScanResponse - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return Result{}, fmt.Errorf("decoding scanner %q response: %w", s.name, err) - } - - return Result{ - Allowed: out.Allowed, - Reason: out.Reason, - Findings: out.Findings, - }, nil -} diff --git a/internal/scanner/http_test.go b/internal/scanner/http_test.go deleted file mode 100644 index 7597826..0000000 --- a/internal/scanner/http_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package scanner - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - "time" -) - -func TestHTTPScanner_Scan(t *testing.T) { - tests := []struct { - name string - respStatus int - respBody string - wantAllowed bool - wantReason string - wantErr bool - }{ - { - name: "allowed", - respStatus: http.StatusOK, - respBody: `{"allowed": true}`, - wantAllowed: true, - }, - { - name: "blocked with reason and findings", - respStatus: http.StatusOK, - respBody: `{"allowed": false, "reason": "malware detected", "findings": [{"severity": "high", "title": "EICAR", "description": "test signature"}]}`, - wantAllowed: false, - wantReason: "malware detected", - }, - { - name: "non-200 status is an error", - respStatus: http.StatusInternalServerError, - respBody: `{}`, - wantErr: true, - }, - { - name: "malformed JSON is an error", - respStatus: http.StatusOK, - respBody: `not json`, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var got Request - if err := json.NewDecoder(r.Body).Decode(&got); err != nil { - t.Errorf("decode request body: %v", err) - } - if got.Ecosystem != "npm" || got.Name != "left-pad" || got.FetchURL == "" { - t.Errorf("unexpected request body: %+v", got) - } - if r.Header.Get("Authorization") != "Bearer secret" { - t.Errorf("missing/incorrect Authorization header: %q", r.Header.Get("Authorization")) - } - - w.WriteHeader(tt.respStatus) - _, _ = w.Write([]byte(tt.respBody)) - })) - defer srv.Close() - - s := NewHTTPScanner("test", srv.URL, map[string]string{"Authorization": "Bearer secret"}, nil) - - result, err := s.Scan(context.Background(), Request{ - Ecosystem: "npm", - Name: "left-pad", - Version: "1.0.0", - FetchURL: srv.URL + "/fetch", - }) - - if tt.wantErr { - if err == nil { - t.Fatalf("Scan() error = nil, want error") - } - return - } - if err != nil { - t.Fatalf("Scan() unexpected error: %v", err) - } - if result.Allowed != tt.wantAllowed { - t.Errorf("Allowed = %v, want %v", result.Allowed, tt.wantAllowed) - } - if result.Reason != tt.wantReason { - t.Errorf("Reason = %q, want %q", result.Reason, tt.wantReason) - } - }) - } -} - -func TestHTTPScanner_Scan_ContextTimeout(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(50 * time.Millisecond) - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"allowed": true}`)) - })) - defer srv.Close() - - s := NewHTTPScanner("slow", srv.URL, nil, nil) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond) - defer cancel() - - _, err := s.Scan(ctx, Request{Ecosystem: "npm", Name: "left-pad"}) - if err == nil { - t.Fatal("Scan() error = nil, want timeout error") - } -} - -func TestHTTPScanner_Name(t *testing.T) { - s := NewHTTPScanner("clamav", "http://example.invalid", nil, nil) - if got := s.Name(); got != "clamav" { - t.Errorf("Name() = %q, want %q", got, "clamav") - } -} diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go deleted file mode 100644 index 380bb4d..0000000 --- a/internal/scanner/scanner.go +++ /dev/null @@ -1,65 +0,0 @@ -// Package scanner provides pluggable pre-cache artifact scanning. -// -// A Scanner inspects an artifact staged in the proxy's own storage before -// it becomes visible to clients, and returns a verdict on whether it may -// be cached. The proxy never uploads artifact bytes to a scanner directly: -// it hands the scanner a short-lived signed URL and the scanner pulls the -// bytes itself. See HTTPScanner for the built-in adapter that implements -// this over a small HTTP/JSON contract, letting trivy, ClamAV, Wiz, or any -// custom service integrate without the proxy needing built-in knowledge of -// any specific tool. -package scanner - -import "context" - -// Request describes a staged artifact awaiting a scan verdict. -type Request struct { - Ecosystem string `json:"ecosystem"` - Name string `json:"name"` - Version string `json:"version"` - Filename string `json:"filename"` - PURL string `json:"purl"` - ContentType string `json:"content_type"` - Size int64 `json:"size"` - - // FetchURL is a short-lived signed URL the scanner must GET itself to - // retrieve the exact bytes staged in the proxy's storage. - FetchURL string `json:"fetch_url"` -} - -// Finding describes a single issue reported by a scanner. -type Finding struct { - Severity string - Title string - Description string -} - -// Result is a scanner's verdict for a Request. -type Result struct { - Allowed bool - Reason string - Findings []Finding - - // ScannerName identifies which scanner produced this result. Set by - // Group, not by individual Scanner implementations. - ScannerName string - - // InfraError reports whether Allowed: false was forced by a scanner - // call failing (network error, timeout, bad response) rather than an - // actual verdict from the scanner. Set by Group. Callers that surface - // Reason to untrusted clients must not do so when this is true: it may - // contain raw connection errors (internal hostnames, ports) instead of - // a verdict meant to be shown outside the proxy. - InfraError bool -} - -// Scanner is the extension point for pluggable pre-cache scanning. -type Scanner interface { - // Name identifies this scanner in logs and metrics. - Name() string - - // Scan requests a verdict for req. Implementations must respect ctx - // cancellation: Group cancels in-flight scans once a blocking verdict - // has already been decided by another scanner. - Scan(ctx context.Context, req Request) (Result, error) -} diff --git a/internal/server/analytics.go b/internal/server/analytics.go deleted file mode 100644 index 2e0e0ce..0000000 --- a/internal/server/analytics.go +++ /dev/null @@ -1,290 +0,0 @@ -package server - -import ( - "fmt" - "net/http" - "strconv" - "strings" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/metrics" -) - -// analyticsTopSources caps how many callers the page lists; the rest are -// summarised in the overflow row, and the access log has every one of them. -const analyticsTopSources = 15 - -// AnalyticsData contains data for rendering the analytics dashboard. -type AnalyticsData struct { - Layout - Totals AnalyticsTotals - Donut DonutView - EnrichmentStats EnrichmentStatsView - Ecosystems []EcosystemRow - Metadata MetadataCacheView - Runtime RuntimeView - // StatsFailed records that the per-ecosystem query errored, so the page - // can say the figures are unavailable instead of claiming an empty cache. - StatsFailed bool - // StatsStale is set when the query errored but a previous snapshot was - // retained and is being shown. Without it a database that has been down - // for an hour renders hour-old figures as current, with the failure - // visible only in the logs. - StatsStale bool - // StatsAge is when the retained snapshot was read, phrased for display. - StatsAge string -} - -// AnalyticsTotals holds the headline figures across every ecosystem. -type AnalyticsTotals struct { - DownloadedBytes int64 - Downloaded string - Downloads string - CacheSize string - CachedArtifacts string - Packages string - Versions string - Ecosystems int - ActiveEcosystems int - // Amplification is accumulated download volume divided by the bytes - // currently held in cache: how many times over the cache has served - // what it stores. Empty when nothing is cached. - Amplification string -} - -// EcosystemRow is one ecosystem's row in the analytics table. -type EcosystemRow struct { - Ecosystem string - DownloadedBytes int64 - Downloaded string - Downloads string - CacheSize string - AvgArtifactSize string - Artifacts string - Packages string - Versions string - // SharePct is this ecosystem's share of the accumulated download total. - SharePct string -} - -// MetadataCacheView summarises the metadata cache, which is stored separately -// from artifacts and so appears in none of the artifact figures. -type MetadataCacheView struct { - // Enabled reports whether cache_metadata is on. Entries can remain from - // an earlier run with it on, so both are shown independently. - Enabled bool - // Failed records that the query errored, so the card can say so rather - // than claim the cache is empty. - Failed bool - Entries string - Size string - Rows []MetadataCacheRow -} - -// MetadataCacheRow is one ecosystem's share of the metadata cache. -type MetadataCacheRow struct { - Ecosystem string - Entries string - Size string - SharePct string -} - -// metadataCacheView turns per-ecosystem metadata cache rows into the card's -// totals and table. -func metadataCacheView(stats []database.MetadataCacheStats, enabled bool) MetadataCacheView { - var entries, size int64 - for _, e := range stats { - entries += e.Entries - size += e.Size - } - - v := MetadataCacheView{ - Enabled: enabled, - Entries: formatCount(entries), - Size: formatSize(size), - Rows: make([]MetadataCacheRow, 0, len(stats)), - } - for _, e := range stats { - v.Rows = append(v.Rows, MetadataCacheRow{ - Ecosystem: e.Ecosystem, - Entries: formatCount(e.Entries), - Size: formatSize(e.Size), - SharePct: formatPercent(e.Size, size), - }) - } - return v -} - -func formatCount(n int64) string { - s := strconv.FormatInt(n, 10) - neg := strings.HasPrefix(s, "-") - if neg { - s = s[1:] - } - - var b strings.Builder - for i, digit := range s { - if i > 0 && (len(s)-i)%3 == 0 { - b.WriteByte(',') - } - b.WriteRune(digit) - } - if neg { - return "-" + b.String() - } - return b.String() -} - -// percentOf renders part/whole as a percentage with one decimal place, always -// as a bare number. formatPercent wraps it to catch the case where a real -// share rounds away to zero. -func percentOf(part, whole int64) string { - if whole <= 0 { - return "0.0" - } - return strconv.FormatFloat(float64(part)/float64(whole)*100, 'f', 1, 64) //nolint:mnd // percent -} - -// formatPercent renders a share for display. A share that is real but rounds -// to zero reads as "<0.1" rather than "0.0", which would claim the ecosystem -// served nothing. -func formatPercent(part, whole int64) string { - pct := percentOf(part, whole) - if pct == "0.0" && part > 0 && whole > 0 { - return "<0.1" - } - return pct -} - -// formatRatio renders an "N.Nx" multiplier, or "" when the denominator is zero. -func formatRatio(numerator, denominator int64) string { - if denominator <= 0 { - return "" - } - return fmt.Sprintf("%.1fx", float64(numerator)/float64(denominator)) -} - -func avgArtifactSize(cacheSize, artifacts int64) string { - if artifacts <= 0 { - return "0 B" - } - return formatSize(cacheSize / artifacts) -} - -// enrichmentStatsView maps enrichment totals onto the shared view struct used -// by both the dashboard and the analytics page. -func enrichmentStatsView(stats *database.EnrichmentStats) EnrichmentStatsView { - return EnrichmentStatsView{ - EnrichedPackages: stats.EnrichedPackages, - VulnSyncedPackages: stats.VulnSyncedPackages, - TotalVulnerabilities: stats.TotalVulnerabilities, - CriticalVulns: stats.CriticalVulns, - HighVulns: stats.HighVulns, - MediumVulns: stats.MediumVulns, - LowVulns: stats.LowVulns, - HasVulns: stats.TotalVulnerabilities > 0, - } -} - -// handleAnalytics renders the analytics dashboard: accumulated download volume -// in total and per ecosystem, alongside the cache and enrichment figures the -// main dashboard reports. -func (s *Server) handleAnalytics(w http.ResponseWriter, r *http.Request) { - ecosystems, err := s.ecoStats.Get(s.db) - statsFailed := err != nil - if err != nil { - s.logger.Error("failed to get ecosystem stats", "error", err) - } - - enrichStats, err := s.db.GetEnrichmentStats() - if err != nil { - s.logger.Error("failed to get enrichment stats", "error", err) - enrichStats = &database.EnrichmentStats{} - } - - data := AnalyticsData{ - Layout: s.layoutFor(r), - EnrichmentStats: enrichmentStatsView(enrichStats), - } - data.StatsFailed = statsFailed - data.StatsStale = statsFailed && len(ecosystems) > 0 - if data.StatsStale { - data.StatsAge = formatTimeAgo(s.ecoStats.SnapshotAt()) - } - data.Totals, data.Ecosystems = analyticsView(ecosystems) - data.Donut = donutView(ecosystems, data.Totals.DownloadedBytes, data.Totals.Downloaded) - - // The same 60-second snapshot the metadata cache gauges are published - // from, so a page load costs no query of its own. A failed read keeps the - // last good rows, which the card shows in preference to the error. - metadataStats, err := s.metaStats.Get(s.db) - if err != nil { - s.logger.Error("failed to get metadata cache stats", "error", err) - } - data.Metadata = metadataCacheView(metadataStats, s.cfg != nil && s.cfg.CacheMetadata) - data.Metadata.Failed = err != nil - - // Process-lifetime counters come from the Prometheus registry rather than - // the database; a failure here must not take the page down with it. - snap, err := metrics.Gather() - if err != nil { - s.logger.Error("failed to gather runtime metrics", "error", err) - } else { - data.Runtime = runtimeView(snap, s.cfg != nil && s.cfg.UIFailureDetails) - if s.cfg != nil && s.cfg.UIRequestSources { - data.Runtime.SourcesOn = true - data.Runtime.Sources = s.sources.Top(analyticsTopSources) - data.Runtime.SourceCount = s.sources.Count() - data.Runtime.TrustsForward = s.trustsForwardedFor() - } - } - - if err := s.templates.Render(w, "analytics", data); err != nil { - s.logger.Error("failed to render analytics", "error", err) - } -} - -// analyticsView turns per-ecosystem database rows into the rendered totals and -// table rows, preserving the order they arrive in. -func analyticsView(stats []database.EcosystemStats) (AnalyticsTotals, []EcosystemRow) { - var totals AnalyticsTotals - var downloads, cacheSize, artifacts, packages, versions int64 - - for _, e := range stats { - totals.DownloadedBytes += e.DownloadedBytes - downloads += e.Downloads - cacheSize += e.CacheSize - artifacts += e.Artifacts - packages += e.Packages - versions += e.Versions - if e.DownloadedBytes > 0 { - totals.ActiveEcosystems++ - } - } - - totals.Downloaded = formatSize(totals.DownloadedBytes) - totals.Downloads = formatCount(downloads) - totals.CacheSize = formatSize(cacheSize) - totals.CachedArtifacts = formatCount(artifacts) - totals.Packages = formatCount(packages) - totals.Versions = formatCount(versions) - totals.Ecosystems = len(stats) - totals.Amplification = formatRatio(totals.DownloadedBytes, cacheSize) - - rows := make([]EcosystemRow, 0, len(stats)) - for _, e := range stats { - rows = append(rows, EcosystemRow{ - Ecosystem: e.Ecosystem, - DownloadedBytes: e.DownloadedBytes, - Downloaded: formatSize(e.DownloadedBytes), - Downloads: formatCount(e.Downloads), - CacheSize: formatSize(e.CacheSize), - AvgArtifactSize: avgArtifactSize(e.CacheSize, e.Artifacts), - Artifacts: formatCount(e.Artifacts), - Packages: formatCount(e.Packages), - Versions: formatCount(e.Versions), - SharePct: formatPercent(e.DownloadedBytes, totals.DownloadedBytes), - }) - } - return totals, rows -} diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go deleted file mode 100644 index 02e67e2..0000000 --- a/internal/server/analytics_coverage_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package server - -import ( - "strings" - "testing" - - "github.com/git-pkgs/proxy/internal/metrics" - "github.com/prometheus/client_golang/prometheus" -) - -// metricSurface records where each registered metric is shown on /ui/analytics. -// -// This is a standing obligation, not a snapshot: registering a new metric -// anywhere in internal/metrics fails the build here until it has both a tile on -// the page and an entry below. That is the point -- the page is meant to be a -// complete view of /metrics -- but it is a cost paid by every future metric, so -// it is written down rather than discovered. -var metricSurface = map[string]string{ - // Database-derived, shown in the donut, the KPI row and the breakdown table. - "proxy_ecosystem_downloaded_bytes": "donut + breakdown table", - "proxy_ecosystem_artifact_downloads": "Downloads tile + breakdown table", - "proxy_ecosystem_cache_size_bytes": "breakdown table", - "proxy_ecosystem_cached_artifacts": "breakdown table", - "proxy_ecosystem_packages": "breakdown table", - "proxy_ecosystem_versions": "breakdown table", - "proxy_cache_size_bytes": "Cache size tile", - "proxy_cached_artifacts_total": "Cached artifacts tile", - "proxy_metadata_cache_size_bytes": "Metadata cache card", - "proxy_metadata_cache_entries": "Metadata cache card", - - // Request-path counters. - "proxy_response_bytes_total": "Runtime: Served", - "proxy_client_requests_total": "Runtime: Request sources -- By client", - "proxy_client_response_bytes_total": "Runtime: Request sources -- By client", - - // Registry-derived, shown in the Runtime card. - "proxy_requests_total": "Runtime: Requests + Responses by status", - "proxy_request_duration_seconds": "Runtime: Mean latency", - "proxy_active_requests": "Runtime: In flight", - "proxy_cache_hits_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", - "proxy_cache_misses_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", - "proxy_metadata_cache_served_bytes_total": "Runtime: Metadata cache lookups -- served from cache", - "proxy_upstream_fetch_duration_seconds": "Runtime: Upstream fetches + mean fetch", - "proxy_upstream_errors_total": "Runtime: Upstream errors", - "proxy_storage_operation_duration_seconds": "Runtime: Storage operations", - "proxy_storage_errors_total": "Runtime: Storage errors", - "proxy_integrity_failures_total": "Runtime: Integrity failures", - "proxy_health_probe_failures_total": "Runtime: Health probe failures", - "proxy_circuit_breaker_state": "Runtime: Circuit breakers", - "proxy_circuit_breaker_trips_total": "Runtime: Circuit breaker trips", - "proxy_artifacts_evicted_total": "Runtime: Evicted artifacts", - "proxy_metadata_evicted_total": "Runtime: Evicted metadata", - "proxy_metadata_expired_by_change_feed_total": "Runtime: Metadata changed upstream", - "proxy_scan_duration_seconds": "Runtime: Pre-cache scanning -- Scans", - "proxy_scan_blocked_total": "Runtime: Pre-cache scanning -- Artifacts blocked", - "proxy_scan_errors_total": "Runtime: Pre-cache scanning -- Scan errors", -} - -// TestEveryMetricIsSurfaced fails when a metric is registered but has no home -// on the analytics page. The page is meant to be a complete view of what -// /metrics exposes, so a silently unsurfaced metric is a gap, not a detail. -func TestEveryMetricIsSurfaced(t *testing.T) { - // Touch every metric family so it is present in the registry output, since - // a vector with no observed label values gathers as nothing at all. - metrics.RecordRequest("npm", 200, 0) - metrics.RecordResponse("npm", "npm", 1) - metrics.RecordArtifactCacheHit("npm") - metrics.RecordArtifactCacheMiss("npm") - metrics.RecordMetadataCacheHit("npm", 1) - metrics.RecordMetadataCacheMiss("npm") - metrics.RecordUpstreamFetch("npm", 0) - metrics.RecordUpstreamError("npm", "fetch_failed", metrics.FailureDetail{}) - metrics.RecordStorageOperation("get", 0) - metrics.RecordStorageError("get", metrics.FailureDetail{}) - metrics.RecordIntegrityFailure("npm", metrics.FailureDetail{}) - metrics.RecordHealthProbeFailure("write", metrics.FailureDetail{}) - metrics.UpdateCircuitBreakerState("example.test", 0) - metrics.RecordCircuitBreakerTrip("example.test") - metrics.RecordScanResult("npm", "clamav", false, 0) - metrics.RecordScanError("npm", "clamav", "timeout", metrics.FailureDetail{}) - metrics.UpdateCacheStats(1, 1) - metrics.RecordArtifactEvicted("retention", "npm") - metrics.RecordMetadataEvicted("npm") - metrics.RecordMetadataExpiredByChangeFeed("npm", 1) - metrics.UpdateEcosystemStats([]metrics.EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 1}}) - metrics.UpdateMetadataCacheStats([]metrics.MetadataCacheStats{{Ecosystem: "npm", Entries: 1, Size: 1}}) - - families, err := prometheus.DefaultGatherer.Gather() - if err != nil { - t.Fatalf("Gather: %v", err) - } - - var registered []string - for _, family := range families { - name := family.GetName() - if strings.HasPrefix(name, "proxy_") { - registered = append(registered, name) - } - } - if len(registered) == 0 { - t.Fatal("no proxy_ metrics in the registry; the probe above is not working") - } - - for _, name := range registered { - if _, ok := metricSurface[name]; !ok { - t.Errorf("%s is registered but has no home on /ui/analytics; "+ - "surface it and add it to metricSurface", name) - } - } - - for name := range metricSurface { - found := false - for _, r := range registered { - if r == name { - found = true - break - } - } - if !found { - t.Errorf("metricSurface lists %s, but it is not registered any more; drop the entry", name) - } - } -} diff --git a/internal/server/analytics_donut.go b/internal/server/analytics_donut.go deleted file mode 100644 index 59abc85..0000000 --- a/internal/server/analytics_donut.go +++ /dev/null @@ -1,208 +0,0 @@ -package server - -import ( - "math" - "strconv" - - "github.com/git-pkgs/proxy/internal/database" -) - -// Donut geometry, in SVG user units. The ring is drawn as a dashed stroke on a -// circle rather than as arc paths: one dash per slice, which makes the 2px -// surface gap between slices fall out of the dash arithmetic. -const ( - donutSize = 260.0 - donutRadius = 104.0 - donutStroke = 30.0 - // donutGap is the surface-coloured separator between touching slices. - donutGap = 2.0 - // donutMinSlice keeps a slice that rounds to nothing from vanishing - // entirely; the legend and table carry its real value. - donutMinSlice = 1.5 - donutCircumference = 2 * math.Pi * donutRadius //nolint:mnd // circumference -) - -// donutSlices is the most slices the ring will draw. Part-to-whole reads at a -// glance only while the segment count stays small, so past this the tail is -// folded into a single "Other" slice and the full detail lives in the table -// below the chart. -const donutSlices = 6 - -const otherSliceLabel = "Other" - -// DonutSlice is one arc of the ring plus its legend row. -type DonutSlice struct { - // Index selects the fixed categorical slot. The hues themselves live in - // the .donut-slot-N and .swatch-N rules in the analytics template, so the - // validated palette has exactly one definition; slots are assigned here in - // sequence and never cycled. - Index int - Label string - Value string - SharePct string - Downloads string - CacheSize string - Dash string - Gap string - Offset string - // IsOther marks the folded tail, which has no single ecosystem behind it. - IsOther bool - // Members lists what was folded in, for the tooltip. - Members string -} - -// DonutView is the whole chart: the ring, its centre figure and its legend. -type DonutView struct { - Slices []DonutSlice - // Size, Radius, Stroke and Center are handed to the template so the SVG - // geometry has exactly one definition. - Size string - Radius string - Stroke string - Center string - // CenterValue is the accumulated download size across every ecosystem, - // stated in the middle of the ring. - CenterValue string - CenterLabel string - HasSlices bool -} - -// donutView folds the per-ecosystem rows into at most donutSlices arcs and -// computes the dash geometry for each. -func donutView(stats []database.EcosystemStats, totalBytes int64, centerValue string) DonutView { - view := DonutView{ - Size: trimFloat(donutSize), - Radius: trimFloat(donutRadius), - Stroke: trimFloat(donutStroke), - Center: trimFloat(donutSize / 2), //nolint:mnd // the centre of the viewBox - CenterValue: centerValue, - CenterLabel: "accumulated download size", - } - - // Only ecosystems that have actually served bytes get an arc; an - // ecosystem at zero would be an invisible slice with a legend entry - // claiming a share it does not have. - active := make([]database.EcosystemStats, 0, len(stats)) - for _, e := range stats { - if e.DownloadedBytes > 0 { - active = append(active, e) - } - } - if len(active) == 0 || totalBytes <= 0 { - return view - } - - head, tail := active, []database.EcosystemStats(nil) - if len(active) > donutSlices { - head, tail = active[:donutSlices-1], active[donutSlices-1:] - } - - slices := make([]DonutSlice, 0, donutSlices) - for i, e := range head { - slices = append(slices, DonutSlice{ - Index: i, - Label: ecosystemBadgeLabel(e.Ecosystem), - Value: formatSize(e.DownloadedBytes), - SharePct: formatPercent(e.DownloadedBytes, totalBytes), - Downloads: formatCount(e.Downloads), - CacheSize: formatSize(e.CacheSize), - }) - } - - if len(tail) > 0 { - var bytes, downloads, cacheSize int64 - members := "" - for i, e := range tail { - bytes += e.DownloadedBytes - downloads += e.Downloads - cacheSize += e.CacheSize - if i > 0 { - members += ", " - } - members += ecosystemBadgeLabel(e.Ecosystem) - } - slices = append(slices, DonutSlice{ - Index: donutSlices - 1, - Label: otherSliceLabel, - Value: formatSize(bytes), - SharePct: formatPercent(bytes, totalBytes), - Downloads: formatCount(downloads), - CacheSize: formatSize(cacheSize), - IsOther: true, - Members: members, - }) - } - - applyDonutGeometry(slices, active, tail, totalBytes) - view.Slices = slices - view.HasSlices = true - return view -} - -// applyDonutGeometry sets each slice's dash length and offset. -func applyDonutGeometry(slices []DonutSlice, active, tail []database.EcosystemStats, totalBytes int64) { - // Recover each slice's byte value in the same order the slices were built, - // so the geometry is driven by the numbers rather than by the rendered text. - values := make([]int64, 0, len(slices)) - head := active - if len(tail) > 0 { - head = active[:len(active)-len(tail)] - } - for _, e := range head { - values = append(values, e.DownloadedBytes) - } - if len(tail) > 0 { - var sum int64 - for _, e := range tail { - sum += e.DownloadedBytes - } - values = append(values, sum) - } - - dashes := make([]float64, len(slices)) - widest := 0 - for i := range slices { - arc := donutCircumference * float64(values[i]) / float64(totalBytes) - - dashes[i] = arc - donutGap - if dashes[i] < donutMinSlice { - dashes[i] = donutMinSlice - } - if dashes[i] > dashes[widest] { - widest = i - } - } - - // Widening a sliver to donutMinSlice buys visibility with room the ring - // does not have, and the overshoot would wrap the last slice back over the - // first. The widest slice gives the space back: it is the only one that can - // lose a couple of units without becoming unreadable, and at six slices the - // most that can be owed is donutSlices*(donutMinSlice+donutGap), far less - // than the circumference. - var needed float64 - for _, d := range dashes { - needed += d + donutGap - } - if excess := needed - donutCircumference; excess > 0 { - dashes[widest] = math.Max(donutMinSlice, dashes[widest]-excess) - } - - var consumed float64 - for i := range slices { - slices[i].Dash = trimFloat(dashes[i]) - slices[i].Gap = trimFloat(donutCircumference - dashes[i]) - // A dashoffset runs backwards around the circle, so the running total - // is negated to lay slices out clockwise from twelve o'clock. - slices[i].Offset = trimFloat(-consumed) - - // Advance by what is drawn: advancing by the smaller true arc would - // start the next slice underneath a widened one, and the later colour - // would win. - consumed += dashes[i] + donutGap - } -} - -// trimFloat renders an SVG coordinate without trailing zeroes. -func trimFloat(f float64) string { - return strconv.FormatFloat(f, 'f', -1, 64) -} diff --git a/internal/server/analytics_donut_test.go b/internal/server/analytics_donut_test.go deleted file mode 100644 index d2e8738..0000000 --- a/internal/server/analytics_donut_test.go +++ /dev/null @@ -1,211 +0,0 @@ -package server - -import ( - "math" - "strconv" - "testing" - - "github.com/git-pkgs/proxy/internal/database" -) - -func eco(name string, bytes int64) database.EcosystemStats { - return database.EcosystemStats{Ecosystem: name, DownloadedBytes: bytes, Downloads: 1, CacheSize: bytes / 2} -} - -func TestDonutViewFoldsTailIntoOther(t *testing.T) { - // Nine active ecosystems: five keep their own slice, four fold into "Other". - stats := []database.EcosystemStats{ - eco("npm", 900), eco("pypi", 800), eco("maven", 700), eco("cargo", 600), - eco("gem", 500), eco("golang", 40), eco("nuget", 30), eco("deb", 20), eco("conda", 10), - } - var total int64 - for _, e := range stats { - total += e.DownloadedBytes - } - - view := donutView(stats, total, formatSize(total)) - - if len(view.Slices) != donutSlices { - t.Fatalf("expected %d slices, got %d", donutSlices, len(view.Slices)) - } - - last := view.Slices[donutSlices-1] - if !last.IsOther { - t.Fatalf("expected the final slice to be the folded tail, got %+v", last) - } - // 40 + 30 + 20 + 10 = 100 - if last.Value != formatSize(100) { - t.Errorf("Other value = %q, want %q", last.Value, formatSize(100)) - } - if last.Members != "golang, nuget, debian, conda" { - t.Errorf("Other members = %q, want the four folded ecosystems by display name", last.Members) - } - // "Other" must take the last slot, never one of a real ecosystem's. - if last.Index != donutSlices-1 { - t.Errorf("Other took slot %d, want %d", last.Index, donutSlices-1) - } - for i, s := range view.Slices[:donutSlices-1] { - if s.Index != i { - t.Errorf("slice %d took slot %d; slots must be assigned in sequence", i, s.Index) - } - } -} - -func TestDonutViewKeepsEveryEcosystemWhenFewEnough(t *testing.T) { - stats := []database.EcosystemStats{eco("npm", 600), eco("pypi", 400)} - view := donutView(stats, 1000, "1000 B") - - if len(view.Slices) != 2 { - t.Fatalf("expected 2 slices, got %d", len(view.Slices)) - } - for _, s := range view.Slices { - if s.IsOther { - t.Errorf("nothing should be folded with only 2 ecosystems, got %+v", s) - } - } - if view.Slices[0].SharePct != "60.0" || view.Slices[1].SharePct != "40.0" { - t.Errorf("shares = %q / %q, want 60.0 / 40.0", view.Slices[0].SharePct, view.Slices[1].SharePct) - } -} - -// The ring must account for the whole circle: the arcs, plus one gap each, -// have to add back up to the circumference. -func TestDonutGeometryCoversTheCircle(t *testing.T) { - stats := []database.EcosystemStats{eco("npm", 500), eco("pypi", 300), eco("maven", 200)} - view := donutView(stats, 1000, "1000 B") - - circumference := 2 * math.Pi * donutRadius - - var covered float64 - for _, s := range view.Slices { - dash, err := strconv.ParseFloat(s.Dash, 64) - if err != nil { - t.Fatalf("dash %q: %v", s.Dash, err) - } - gap, err := strconv.ParseFloat(s.Gap, 64) - if err != nil { - t.Fatalf("gap %q: %v", s.Gap, err) - } - // Each slice is drawn as one dash followed by a gap spanning the - // rest of the circle, so the pair always sums to the circumference. - if math.Abs(dash+gap-circumference) > 0.01 { - t.Errorf("slice %q: dash+gap = %v, want the circumference %v", s.Label, dash+gap, circumference) - } - covered += dash + donutGap - } - - if math.Abs(covered-circumference) > 0.01 { - t.Errorf("arcs plus gaps cover %v, want the full circumference %v", covered, circumference) - } -} - -// Offsets must advance monotonically so slices sit end to end instead of -// stacking on top of each other. -func TestDonutOffsetsAdvanceInOrder(t *testing.T) { - stats := []database.EcosystemStats{eco("npm", 500), eco("pypi", 300), eco("maven", 200)} - view := donutView(stats, 1000, "1000 B") - - circumference := 2 * math.Pi * donutRadius - want := []float64{0, -circumference * 0.5, -circumference * 0.8} - - for i, s := range view.Slices { - got, err := strconv.ParseFloat(s.Offset, 64) - if err != nil { - t.Fatalf("offset %q: %v", s.Offset, err) - } - if math.Abs(got-want[i]) > 0.01 { - t.Errorf("slice %d offset = %v, want %v", i, got, want[i]) - } - } -} - -// A slice too small to render still has to be visible; the legend and table -// carry its real value. -func TestDonutTinySliceStaysVisible(t *testing.T) { - stats := []database.EcosystemStats{eco("npm", 10_000_000), eco("composer", 1)} - view := donutView(stats, 10_000_001, "9.5 MB") - - tiny := view.Slices[1] - dash, err := strconv.ParseFloat(tiny.Dash, 64) - if err != nil { - t.Fatalf("dash %q: %v", tiny.Dash, err) - } - if dash < donutMinSlice { - t.Errorf("tiny slice dash = %v, want at least %v so it stays visible", dash, donutMinSlice) - } - if tiny.SharePct != "<0.1" { - t.Errorf("tiny slice share = %q, want %q", tiny.SharePct, "<0.1") - } -} - -func TestDonutViewEmpty(t *testing.T) { - if view := donutView(nil, 0, "0 B"); view.HasSlices || len(view.Slices) != 0 { - t.Errorf("expected an empty ring, got %+v", view) - } - - // Ecosystems that exist but have served nothing get no arc at all. - idle := []database.EcosystemStats{{Ecosystem: "rpm", Packages: 3}} - if view := donutView(idle, 0, "0 B"); view.HasSlices { - t.Errorf("an ecosystem with no traffic must not get a slice, got %+v", view.Slices) - } -} - -// A slice widened to stay visible must not be drawn over by its successor: -// the next offset has to start where this slice actually ends. -func TestDonutClampedSliceDoesNotOverdrawItsSuccessor(t *testing.T) { - // Four slivers far below the minimum, followed by a large slice. - stats := []database.EcosystemStats{ - eco("npm", 10_000_000), eco("a", 1), eco("b", 1), eco("c", 1), eco("d", 1), - } - view := donutView(stats, 10_000_004, "9.5 MB") - - for i := 0; i < len(view.Slices)-1; i++ { - dash := mustFloat(t, view.Slices[i].Dash) - start := -mustFloat(t, view.Slices[i].Offset) - nextStart := -mustFloat(t, view.Slices[i+1].Offset) - - end := start + dash - if nextStart < end-0.001 { - t.Errorf("slice %d (%q) is drawn to %v but slice %d starts at %v: they overlap", - i, view.Slices[i].Label, end, i+1, nextStart) - } - } -} - -func mustFloat(t *testing.T, s string) float64 { - t.Helper() - f, err := strconv.ParseFloat(s, 64) - if err != nil { - t.Fatalf("parsing %q: %v", s, err) - } - return f -} - -// Widening slivers must not push the ring past a full turn: the last slice -// would wrap back over the first and repaint the leading ecosystem's arc. -func TestDonutNeverExceedsOneTurn(t *testing.T) { - // One dominant ecosystem and five far below the visible minimum. - stats := []database.EcosystemStats{ - eco("npm", 100_000_000), - eco("a", 1), eco("b", 1), eco("c", 1), eco("d", 1), eco("e", 1), - } - view := donutView(stats, 100_000_005, "95.4 MB") - - if len(view.Slices) != 6 { - t.Fatalf("expected 6 slices, got %d", len(view.Slices)) - } - - last := view.Slices[len(view.Slices)-1] - end := -mustFloat(t, last.Offset) + mustFloat(t, last.Dash) - if end > donutCircumference+0.001 { - t.Errorf("ring is drawn to %v, past the circumference %v: the last slice "+ - "wraps over the first", end, donutCircumference) - } - - // Every sliver still has to be visible. - for _, s := range view.Slices[1:] { - if dash := mustFloat(t, s.Dash); dash < donutMinSlice { - t.Errorf("slice %q dash = %v, want at least %v", s.Label, dash, donutMinSlice) - } - } -} diff --git a/internal/server/analytics_failures_test.go b/internal/server/analytics_failures_test.go deleted file mode 100644 index aa89cb5..0000000 --- a/internal/server/analytics_failures_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package server - -import ( - "errors" - "strings" - "testing" - - "github.com/git-pkgs/proxy/internal/metrics" - "github.com/prometheus/client_golang/prometheus" -) - -func storageErrorSnapshot(t *testing.T, operation string) *metrics.Snapshot { - t.Helper() - reg := prometheus.NewRegistry() - errs := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_storage_errors_total", Help: "t"}, []string{"operation"}) - reg.MustRegister(errs) - errs.WithLabelValues(operation).Inc() - - snap, err := metrics.GatherFrom(reg) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - return snap -} - -func TestRuntimeViewAttachesRecentFailuresWhenEnabled(t *testing.T) { - const operation = "test-attach" - metrics.RecordStorageError(operation, metrics.FailureDetail{ - PURL: "pkg:npm/left-pad@1.0.0", Filename: "left-pad-1.0.0.tgz", Path: "npm/left-pad", Err: errors.New("disk full")}) - - v := runtimeView(storageErrorSnapshot(t, operation), true) - - if !v.FailureDetailsOn { - t.Error("FailureDetailsOn = false, want true") - } - if len(v.StorageErrors) != 1 || len(v.StorageErrors[0].Recent) == 0 { - t.Fatalf("StorageErrors = %+v, want one row with recent failures", v.StorageErrors) - } - got := v.StorageErrors[0].Recent[0] - if got.PURL != "pkg:npm/left-pad@1.0.0" || got.Filename != "left-pad-1.0.0.tgz" || - got.Path != "npm/left-pad" || got.Error != "disk full" || got.Age == "" { - t.Errorf("recent failure = %+v", got) - } -} - -func TestRuntimeViewOmitsRecentFailuresWhenDisabled(t *testing.T) { - const operation = "test-omit" - metrics.RecordStorageError(operation, metrics.FailureDetail{PURL: "pkg:npm/left-pad@1.0.0"}) - - v := runtimeView(storageErrorSnapshot(t, operation), false) - - if v.FailureDetailsOn { - t.Error("FailureDetailsOn = true, want false") - } - if len(v.StorageErrors) != 1 || v.StorageErrors[0].Recent != nil { - t.Errorf("StorageErrors = %+v, want the count without recent failures", v.StorageErrors) - } -} - -// Package names and upstream URLs say what the proxy's users depend on, so like -// the caller table they are published only when asked for. -func TestAnalyticsPageHidesFailureDetailsByDefault(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - - if ts.server.cfg.UIFailureDetails { - t.Fatal("UIFailureDetails defaults to true; it must default off") - } - metrics.RecordUpstreamError("npm", "fetch_failed", metrics.FailureDetail{PURL: "pkg:npm/hidden-by-default@1.0.0"}) - - body := ts.getOK(t, "/ui/analytics") - if strings.Contains(body, "hidden-by-default") { - t.Error("a failed package rendered with ui_failure_details off") - } - if !strings.Contains(body, "ui_failure_details") { - t.Error("the off-state note naming the config key did not render") - } -} - -func TestAnalyticsPageShowsFailureDetailsWhenEnabled(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - - ts.server.cfg.UIFailureDetails = true - metrics.RecordUpstreamError("npm", "fetch_failed", metrics.FailureDetail{ - PURL: "pkg:npm/shown-when-enabled@1.0.0", - URL: "https://registry.example.com/shown-when-enabled.tgz?token=secret", - Err: errors.New("upstream returned 503"), - }) - - body := ts.getOK(t, "/ui/analytics") - for _, want := range []string{ - "pkg:npm/shown-when-enabled@1.0.0", - "https://registry.example.com/shown-when-enabled.tgz", - "upstream returned 503", - } { - if !strings.Contains(body, want) { - t.Errorf("rendered page missing %q", want) - } - } - if strings.Contains(body, "token=secret") { - t.Error("rendered page leaks the URL's query string") - } -} diff --git a/internal/server/analytics_runtime.go b/internal/server/analytics_runtime.go deleted file mode 100644 index 103e33e..0000000 --- a/internal/server/analytics_runtime.go +++ /dev/null @@ -1,375 +0,0 @@ -package server - -import ( - "fmt" - "sort" - "strconv" - "time" - - "github.com/git-pkgs/proxy/internal/metrics" -) - -// RuntimeView holds the process-lifetime figures read out of the Prometheus -// registry. Unlike the cache figures, which are computed from the database and -// survive a restart, everything here starts at zero when the proxy starts. -type RuntimeView struct { - Available bool - - Requests string - ActiveRequests string - RequestMean string - StatusClasses []LabelledCount - // Artifact and metadata lookups are counted apart: they are separate - // caches. Only artifacts feed the database figures above. - ArtifactCache CacheLookups - MetadataCache CacheLookups - MetadataServed string - - UpstreamFetches string - UpstreamFetchMean string - UpstreamErrors []LabelledCount - - StorageOps []LabelledStat - StorageErrors []LabelledCount - IntegrityFailures []LabelledCount - ProbeFailures []LabelledCount - Evictions []LabelledCount - MetadataEvictions []LabelledCount - ChangeFeedExpired []LabelledCount - - Breakers []BreakerRow - BreakerTrips []LabelledCount - - Scans []LabelledStat - ScansBlocked []LabelledCount - ScanErrors []LabelledCount - ScanningOn bool - - ResponseBytes string - Clients []ClientRow - Sources []SourceRow - SourceCount int - TrustsForward bool - // SourcesOn gates the caller table. The page is unauthenticated, so the - // addresses behind it are published only when asked for. - SourcesOn bool - // FailureDetailsOn gates the recent failures listed under each failure - // row, for the same reason as SourcesOn. - FailureDetailsOn bool - // HasFailures is set when any failure counter has fired, which is when the - // note about turning failure details on is worth showing. - HasFailures bool -} - -// CacheLookups is one cache's hits and misses since the process started. -type CacheLookups struct { - Hits string - Misses string - HitRatio string - HasTraffic bool -} - -// cacheLookups reads one kind's share of the hit and miss counters. -func cacheLookups(hits, misses map[string]float64, kind string) CacheLookups { - h, m := hits[kind], misses[kind] - c := CacheLookups{ - Hits: formatCount(int64(h)), - Misses: formatCount(int64(m)), - HasTraffic: h+m > 0, - } - if c.HasTraffic { - c.HitRatio = strconv.FormatFloat(h/(h+m)*100, 'f', 1, 64) //nolint:mnd // percent - } - return c -} - -// ClientRow is one client tool's share of requests and bytes. -type ClientRow struct { - Client string - Requests string - Bytes string -} - -// LabelledCount is a single counter series rendered as a row. -type LabelledCount struct { - Label string - Count string - // Bad marks a failure row, which the template tints. - Bad bool - // Recent lists the latest failures behind the count, newest first. - Recent []FailureRow - // series is the counter series the row was rendered from, for looking up - // what was recorded against it. - series map[string]string -} - -// FailureRow is one recorded failure behind a failure counter. -type FailureRow struct { - Age string - PURL string - Filename string - URL string - Path string - Error string -} - -// LabelledStat is a histogram series rendered as a row: how many observations, -// and their mean. -type LabelledStat struct { - Label string - Count string - Mean string -} - -// BreakerRow is one upstream's circuit breaker state. -type BreakerRow struct { - Registry string - State string - Open bool -} - -// runtimeView shapes a registry snapshot for the analytics page. With -// failureDetails set, each failure row carries the latest failures behind it. -func runtimeView(snap *metrics.Snapshot, failureDetails bool) RuntimeView { - if snap == nil { - return RuntimeView{} - } - - v := RuntimeView{Available: true, FailureDetailsOn: failureDetails} - failures := func(name string, labels ...string) []LabelledCount { - return failureRows(snap, name, failureDetails, labels...) - } - - requests := snap.Sum("proxy_requests_total") - v.Requests = formatCount(int64(requests)) - v.ActiveRequests = formatCount(int64(snap.Sum("proxy_active_requests"))) - v.RequestMean = formatDuration(snap.Mean("proxy_request_duration_seconds")) - v.StatusClasses = statusClasses(snap) - - hits := snap.SumBy("proxy_cache_hits_total", "kind") - misses := snap.SumBy("proxy_cache_misses_total", "kind") - v.ArtifactCache = cacheLookups(hits, misses, metrics.CacheKindArtifact) - v.MetadataCache = cacheLookups(hits, misses, metrics.CacheKindMetadata) - v.MetadataServed = formatSize(int64(snap.Sum("proxy_metadata_cache_served_bytes_total"))) - - v.UpstreamFetches = formatCount(int64(snap.Count("proxy_upstream_fetch_duration_seconds"))) - v.UpstreamFetchMean = formatDuration(snap.Mean("proxy_upstream_fetch_duration_seconds")) - v.UpstreamErrors = failures("proxy_upstream_errors_total", "ecosystem", "error_type") - - v.StorageOps = histogramRows(snap, "proxy_storage_operation_duration_seconds", "operation") - v.StorageErrors = failures("proxy_storage_errors_total", "operation") - v.IntegrityFailures = failures("proxy_integrity_failures_total", "ecosystem") - v.ProbeFailures = failures("proxy_health_probe_failures_total", "step") - v.Evictions = countRows(snap, "proxy_artifacts_evicted_total", "reason", "ecosystem") - v.MetadataEvictions = countRows(snap, "proxy_metadata_evicted_total", "ecosystem") - v.ChangeFeedExpired = countRows(snap, "proxy_metadata_expired_by_change_feed_total", "ecosystem") - - v.Breakers = breakerRows(snap) - v.BreakerTrips = failures("proxy_circuit_breaker_trips_total", "registry") - - v.Scans = histogramRows(snap, "proxy_scan_duration_seconds", "ecosystem", "scanner") - v.ScansBlocked = failures("proxy_scan_blocked_total", "ecosystem", "scanner") - v.ScanErrors = failures("proxy_scan_errors_total", "ecosystem", "scanner", "error_type") - v.ScanningOn = len(v.Scans) > 0 || len(v.ScansBlocked) > 0 || len(v.ScanErrors) > 0 - v.HasFailures = len(v.UpstreamErrors) > 0 || len(v.StorageErrors) > 0 || len(v.IntegrityFailures) > 0 || - len(v.ProbeFailures) > 0 || len(v.ScanErrors) > 0 - - v.ResponseBytes = formatSize(int64(snap.Sum("proxy_response_bytes_total"))) - v.Clients = clientRows(snap) - - return v -} - -// clientRows pairs each client tool's request count with the bytes it pulled, -// busiest first. Both come from the same closed label set, so the two counters -// line up row for row. -func clientRows(snap *metrics.Snapshot) []ClientRow { - bytesByClient := snap.SumBy("proxy_client_response_bytes_total", "client") - - type entry struct { - client string - requests float64 - bytes float64 - } - - entries := make([]entry, 0) - for _, s := range snap.Samples("proxy_client_requests_total") { - client := s.Label("client") - entries = append(entries, entry{ - client: client, - requests: s.Value, - bytes: bytesByClient[client], - }) - } - - sort.Slice(entries, func(i, j int) bool { - a, b := entries[i], entries[j] - if a.bytes != b.bytes { - return a.bytes > b.bytes - } - return a.requests > b.requests - }) - - rows := make([]ClientRow, 0, len(entries)) - for _, e := range entries { - rows = append(rows, ClientRow{ - Client: e.client, - Requests: formatCount(int64(e.requests)), - Bytes: formatSize(int64(e.bytes)), - }) - } - return rows -} - -// statusClasses groups proxy_requests_total into 2xx/3xx/4xx/5xx buckets, which -// is the breakdown worth showing; the per-code detail stays in Prometheus. -func statusClasses(snap *metrics.Snapshot) []LabelledCount { - byClass := make(map[string]float64) - for _, s := range snap.Samples("proxy_requests_total") { - status := s.Label("status") - if len(status) == 0 { - continue - } - byClass[string(status[0])+"xx"] += s.Value - } - - classes := make([]string, 0, len(byClass)) - for c := range byClass { - classes = append(classes, c) - } - sort.Strings(classes) - - rows := make([]LabelledCount, 0, len(classes)) - for _, c := range classes { - rows = append(rows, LabelledCount{ - Label: c, - Count: formatCount(int64(byClass[c])), - Bad: c == "5xx", - }) - } - return rows -} - -// failureRows renders every non-zero series of a failure counter, keyed by the -// joined values of the given labels. Zero-valued series are dropped: a counter -// that has never fired carries no information, and a wall of zeroes buries the -// rows that matter. With withRecent set, each row carries the latest failures -// recorded for its series. -func failureRows(snap *metrics.Snapshot, name string, withRecent bool, labels ...string) []LabelledCount { - rows := countRows(snap, name, labels...) - for i := range rows { - rows[i].Bad = true - if withRecent { - rows[i].Recent = recentFailureRows(metrics.RecentFailures(name, rows[i].series)) - } - } - return rows -} - -// countRows renders every non-zero series of a counter as a row. -func countRows(snap *metrics.Snapshot, name string, labels ...string) []LabelledCount { - var rows []LabelledCount - for _, s := range snap.Samples(name) { - if s.Value == 0 { - continue - } - rows = append(rows, LabelledCount{ - Label: joinLabels(s, labels...), - Count: formatCount(int64(s.Value)), - series: s.Labels, - }) - } - return rows -} - -func recentFailureRows(failures []metrics.Failure) []FailureRow { - if len(failures) == 0 { - return nil - } - rows := make([]FailureRow, 0, len(failures)) - for _, f := range failures { - rows = append(rows, FailureRow{ - Age: formatTimeAgo(f.At), - PURL: f.PURL, - Filename: f.Filename, - URL: f.URL, - Path: f.Path, - Error: f.Error, - }) - } - return rows -} - -// histogramRows renders every observed series of a histogram with its count and mean. -func histogramRows(snap *metrics.Snapshot, name string, labels ...string) []LabelledStat { - var rows []LabelledStat - for _, s := range snap.Samples(name) { - if s.Count == 0 { - continue - } - rows = append(rows, LabelledStat{ - Label: joinLabels(s, labels...), - Count: formatCount(int64(s.Count)), - Mean: formatDuration(s.Mean()), - }) - } - return rows -} - -func breakerRows(snap *metrics.Snapshot) []BreakerRow { - var rows []BreakerRow - for _, s := range snap.Samples("proxy_circuit_breaker_state") { - state := "closed" - switch s.Value { - case 1: - state = "half-open" - case 2: //nolint:mnd // 2 = open, per the gauge's documented encoding - state = "open" - } - rows = append(rows, BreakerRow{ - Registry: s.Label("registry"), - State: state, - Open: s.Value > 0, - }) - } - return rows -} - -func joinLabels(s metrics.Sample, labels ...string) string { - out := "" - for _, l := range labels { - v := s.Label(l) - if v == "" { - continue - } - if out != "" { - out += " · " - } - out += v - } - if out == "" { - return "-" - } - return out -} - -// formatDuration renders a mean latency at a sensible unit. Storage operations -// land in microseconds and upstream fetches in seconds, so a fixed unit would -// read as either 0.000 or an unreadable pile of digits. -func formatDuration(seconds float64) string { - if seconds <= 0 { - return "-" - } - - d := time.Duration(seconds * float64(time.Second)) - switch { - case d < time.Microsecond: - return fmt.Sprintf("%.0f ns", float64(d)) - case d < time.Millisecond: - return fmt.Sprintf("%.0f µs", float64(d)/float64(time.Microsecond)) - case d < time.Second: - return fmt.Sprintf("%.1f ms", float64(d)/float64(time.Millisecond)) - default: - return fmt.Sprintf("%.2f s", d.Seconds()) - } -} diff --git a/internal/server/analytics_runtime_test.go b/internal/server/analytics_runtime_test.go deleted file mode 100644 index 03bc236..0000000 --- a/internal/server/analytics_runtime_test.go +++ /dev/null @@ -1,288 +0,0 @@ -package server - -import ( - "testing" - - "github.com/git-pkgs/proxy/internal/metrics" - "github.com/prometheus/client_golang/prometheus" -) - -func TestFormatDuration(t *testing.T) { - tests := []struct { - seconds float64 - want string - }{ - {0, "-"}, - {-1, "-"}, - {0.0000004, "400 ns"}, - {0.00041, "410 µs"}, - {0.0081, "8.1 ms"}, - {1.25, "1.25 s"}, - } - for _, tc := range tests { - if got := formatDuration(tc.seconds); got != tc.want { - t.Errorf("formatDuration(%v) = %q, want %q", tc.seconds, got, tc.want) - } - } -} - -func TestRuntimeViewNilSnapshot(t *testing.T) { - if v := runtimeView(nil, false); v.Available { - t.Error("a nil snapshot must not report itself as available") - } -} - -func TestRuntimeViewShapesRegistry(t *testing.T) { - reg := prometheus.NewRegistry() - - requests := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_requests_total", Help: "t"}, []string{"ecosystem", "status"}) - scanErrors := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_scan_errors_total", Help: "t"}, []string{"ecosystem", "scanner", "error_type"}) - hits := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_cache_hits_total", Help: "t"}, []string{"ecosystem", "kind"}) - misses := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_cache_misses_total", Help: "t"}, []string{"ecosystem", "kind"}) - breaker := prometheus.NewGaugeVec( - prometheus.GaugeOpts{Name: "proxy_circuit_breaker_state", Help: "t"}, []string{"registry"}) - storage := prometheus.NewHistogramVec( - prometheus.HistogramOpts{Name: "proxy_storage_operation_duration_seconds", Help: "t"}, []string{"operation"}) - reg.MustRegister(requests, scanErrors, hits, misses, breaker, storage) - - requests.WithLabelValues("npm", "200").Add(90) - requests.WithLabelValues("npm", "304").Add(5) - requests.WithLabelValues("npm", "500").Add(5) - scanErrors.WithLabelValues("npm", "clamav", "timeout").Add(3) - hits.WithLabelValues("npm", metrics.CacheKindArtifact).Add(80) - misses.WithLabelValues("npm", metrics.CacheKindArtifact).Add(20) - breaker.WithLabelValues("registry.npmjs.org").Set(0) - breaker.WithLabelValues("static.crates.io").Set(2) - storage.WithLabelValues("get").Observe(0.002) - - snap, err := metrics.GatherFrom(reg) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - v := runtimeView(snap, false) - - if !v.Available { - t.Fatal("expected the view to be available") - } - if v.Requests != "100" { - t.Errorf("Requests = %q, want %q", v.Requests, "100") - } - if a := v.ArtifactCache; a.HitRatio != "80.0" || !a.HasTraffic { - t.Errorf("ArtifactCache = %+v, want an 80.0%% hit rate", a) - } - - // Status codes collapse to classes; 5xx is flagged as a failure row. - wantClasses := map[string]struct { - count string - bad bool - }{"2xx": {"90", false}, "3xx": {"5", false}, "5xx": {"5", true}} - if len(v.StatusClasses) != len(wantClasses) { - t.Fatalf("got %d status classes, want %d: %+v", len(v.StatusClasses), len(wantClasses), v.StatusClasses) - } - for _, row := range v.StatusClasses { - want, ok := wantClasses[row.Label] - if !ok { - t.Errorf("unexpected status class %q", row.Label) - continue - } - if row.Count != want.count || row.Bad != want.bad { - t.Errorf("%s = %q (bad=%v), want %q (bad=%v)", row.Label, row.Count, row.Bad, want.count, want.bad) - } - } - - // The metric that prompted this: it must reach the page with its labels joined. - if len(v.ScanErrors) != 1 { - t.Fatalf("expected 1 scan error row, got %+v", v.ScanErrors) - } - if v.ScanErrors[0].Label != "npm · clamav · timeout" || v.ScanErrors[0].Count != "3" { - t.Errorf("scan error row = %+v, want npm · clamav · timeout = 3", v.ScanErrors[0]) - } - if !v.ScanningOn { - t.Error("scanning should read as on once a scan metric carries a value") - } - - if len(v.Breakers) != 2 { - t.Fatalf("expected 2 breakers, got %+v", v.Breakers) - } - var open, closed int - for _, b := range v.Breakers { - if b.Open { - open++ - if b.State != "open" { - t.Errorf("open breaker state = %q, want %q", b.State, "open") - } - } else { - closed++ - } - } - if open != 1 || closed != 1 { - t.Errorf("breakers: %d open / %d closed, want 1 / 1", open, closed) - } - - if len(v.StorageOps) != 1 || v.StorageOps[0].Label != "get" || v.StorageOps[0].Mean != "2.0 ms" { - t.Errorf("StorageOps = %+v, want one 'get' row with a 2.0 ms mean", v.StorageOps) - } -} - -// Evictions are routine housekeeping, not failures: they reach the page by -// reason and ecosystem without the failure tint. -func TestRuntimeViewShowsEvictions(t *testing.T) { - reg := prometheus.NewRegistry() - evicted := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_artifacts_evicted_total", Help: "t"}, []string{"reason", "ecosystem"}) - reg.MustRegister(evicted) - evicted.WithLabelValues("retention", "npm").Add(4) - evicted.WithLabelValues("lru", "cargo") - - snap, err := metrics.GatherFrom(reg) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - v := runtimeView(snap, false) - - if len(v.Evictions) != 1 { - t.Fatalf("Evictions = %+v, want only the non-zero row", v.Evictions) - } - if got := v.Evictions[0]; got.Label != "retention · npm" || got.Count != "4" || got.Bad { - t.Errorf("eviction row = %+v, want retention · npm = 4 without the failure tint", got) - } -} - -func TestRuntimeViewShowsMetadataEvictions(t *testing.T) { - reg := prometheus.NewRegistry() - evicted := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_metadata_evicted_total", Help: "t"}, []string{"ecosystem"}) - reg.MustRegister(evicted) - evicted.WithLabelValues("golang").Add(3) - - snap, err := metrics.GatherFrom(reg) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - v := runtimeView(snap, false) - - if len(v.MetadataEvictions) != 1 || v.MetadataEvictions[0].Label != "golang" || v.MetadataEvictions[0].Count != "3" { - t.Errorf("MetadataEvictions = %+v, want golang = 3", v.MetadataEvictions) - } -} - -func TestRuntimeViewShowsChangeFeedExpiries(t *testing.T) { - reg := prometheus.NewRegistry() - expired := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_metadata_expired_by_change_feed_total", Help: "t"}, []string{"ecosystem"}) - reg.MustRegister(expired) - expired.WithLabelValues("packagist").Add(2) - - snap, err := metrics.GatherFrom(reg) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - v := runtimeView(snap, false) - - if len(v.ChangeFeedExpired) != 1 || v.ChangeFeedExpired[0].Label != "packagist" || v.ChangeFeedExpired[0].Count != "2" { - t.Errorf("ChangeFeedExpired = %+v, want packagist = 2", v.ChangeFeedExpired) - } -} - -// A counter sitting at zero carries no information and would bury the rows -// that do, so it is left out. -func TestRuntimeViewDropsZeroCounters(t *testing.T) { - reg := prometheus.NewRegistry() - errs := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_storage_errors_total", Help: "t"}, []string{"operation"}) - reg.MustRegister(errs) - - // Touching a label creates the series at zero. - errs.WithLabelValues("get") - errs.WithLabelValues("put").Add(2) - - snap, err := metrics.GatherFrom(reg) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - v := runtimeView(snap, false) - - if len(v.StorageErrors) != 1 { - t.Fatalf("expected only the non-zero row, got %+v", v.StorageErrors) - } - if v.StorageErrors[0].Label != "put" { - t.Errorf("kept row = %q, want %q", v.StorageErrors[0].Label, "put") - } -} - -// Artifact and metadata lookups are separate caches, so the view reports each -// on its own rather than pooling them into one hit rate. -func TestRuntimeViewSplitsCacheKinds(t *testing.T) { - reg := prometheus.NewRegistry() - hits := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_cache_hits_total", Help: "t"}, []string{"ecosystem", "kind"}) - misses := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_cache_misses_total", Help: "t"}, []string{"ecosystem", "kind"}) - served := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_metadata_cache_served_bytes_total", Help: "t"}, []string{"ecosystem"}) - reg.MustRegister(hits, misses, served) - - hits.WithLabelValues("npm", metrics.CacheKindArtifact).Add(80) - misses.WithLabelValues("npm", metrics.CacheKindArtifact).Add(20) - hits.WithLabelValues("pypi", metrics.CacheKindMetadata).Add(20) - hits.WithLabelValues("npm", metrics.CacheKindMetadata).Add(10) - misses.WithLabelValues("pypi", metrics.CacheKindMetadata).Add(10) - served.WithLabelValues("pypi").Add(2048) - - snap, err := metrics.GatherFrom(reg) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - v := runtimeView(snap, false) - - want := map[string]struct { - got CacheLookups - want CacheLookups - }{ - "artifact": {v.ArtifactCache, CacheLookups{Hits: "80", Misses: "20", HitRatio: "80.0", HasTraffic: true}}, - "metadata": {v.MetadataCache, CacheLookups{Hits: "30", Misses: "10", HitRatio: "75.0", HasTraffic: true}}, - } - for kind, c := range want { - if c.got != c.want { - t.Errorf("%s lookups = %+v, want %+v", kind, c.got, c.want) - } - } - if v.MetadataServed != "2.0 KB" { - t.Errorf("MetadataServed = %q, want %q", v.MetadataServed, "2.0 KB") - } -} - -// An empty registry must still render, reporting nothing rather than dividing -// by zero or showing a hit rate of NaN. -func TestRuntimeViewEmptyRegistry(t *testing.T) { - snap, err := metrics.GatherFrom(prometheus.NewRegistry()) - if err != nil { - t.Fatalf("GatherFrom: %v", err) - } - v := runtimeView(snap, false) - - if !v.Available { - t.Error("an empty registry is still a valid snapshot") - } - for name, c := range map[string]CacheLookups{"artifact": v.ArtifactCache, "metadata": v.MetadataCache} { - if c.HasTraffic { - t.Errorf("%s: no traffic recorded, so HasTraffic must be false", name) - } - if c.HitRatio != "" { - t.Errorf("%s: HitRatio = %q, want empty when nothing was recorded", name, c.HitRatio) - } - } - if v.Requests != "0" { - t.Errorf("Requests = %q, want %q", v.Requests, "0") - } - if v.RequestMean != "-" { - t.Errorf("RequestMean = %q, want a dash", v.RequestMean) - } - if v.ScanningOn { - t.Error("scanning must read as off with no scan metrics") - } -} diff --git a/internal/server/analytics_test.go b/internal/server/analytics_test.go deleted file mode 100644 index 540408b..0000000 --- a/internal/server/analytics_test.go +++ /dev/null @@ -1,370 +0,0 @@ -package server - -import ( - "database/sql" - "net/http/httptest" - "strings" - "testing" - - "github.com/git-pkgs/proxy/internal/database" -) - -func TestFormatCount(t *testing.T) { - tests := []struct { - in int64 - want string - }{ - {0, "0"}, - {7, "7"}, - {100, "100"}, - {1000, "1,000"}, - {12004, "12,004"}, - {1000000, "1,000,000"}, - {-4200, "-4,200"}, - } - for _, tc := range tests { - if got := formatCount(tc.in); got != tc.want { - t.Errorf("formatCount(%d) = %q, want %q", tc.in, got, tc.want) - } - } -} - -func TestPercentOf(t *testing.T) { - tests := []struct { - part, whole int64 - want string - }{ - {1, 4, "25.0"}, - {2, 3, "66.7"}, - {0, 100, "0.0"}, - {5, 5, "100.0"}, - // A zero total must not divide by zero. - {0, 0, "0.0"}, - {7, 0, "0.0"}, - // A negligible share still rounds to a plain number here; only - // formatPercent turns it into "<0.1". - {1, 1000000, "0.0"}, - } - for _, tc := range tests { - if got := percentOf(tc.part, tc.whole); got != tc.want { - t.Errorf("percentOf(%d, %d) = %q, want %q", tc.part, tc.whole, got, tc.want) - } - } -} - -func TestFormatPercent(t *testing.T) { - tests := []struct { - part, whole int64 - want string - }{ - {1, 4, "25.0"}, - {0, 100, "0.0"}, - {0, 0, "0.0"}, - // A real but tiny share must not be reported as zero. - {1, 1000000, "<0.1"}, - {7_800_000, 18_000_000_000, "<0.1"}, - } - for _, tc := range tests { - if got := formatPercent(tc.part, tc.whole); got != tc.want { - t.Errorf("formatPercent(%d, %d) = %q, want %q", tc.part, tc.whole, got, tc.want) - } - } -} - -func TestFormatRatio(t *testing.T) { - if got := formatRatio(3500, 1000); got != "3.5x" { - t.Errorf("formatRatio(3500, 1000) = %q, want %q", got, "3.5x") - } - // Nothing cached means no meaningful multiplier, not "+Infx". - if got := formatRatio(3500, 0); got != "" { - t.Errorf("formatRatio(3500, 0) = %q, want empty", got) - } -} - -func TestAvgArtifactSize(t *testing.T) { - if got := avgArtifactSize(1000, 4); got != "250 B" { - t.Errorf("avgArtifactSize(1000, 4) = %q, want %q", got, "250 B") - } - if got := avgArtifactSize(0, 0); got != "0 B" { - t.Errorf("avgArtifactSize(0, 0) = %q, want %q", got, "0 B") - } -} - -func TestAnalyticsView(t *testing.T) { - stats := []database.EcosystemStats{ - {Ecosystem: "npm", Packages: 2, Versions: 2, Artifacts: 2, CacheSize: 1500, Downloads: 4, DownloadedBytes: 3000}, - {Ecosystem: "cargo", Packages: 1, Versions: 1, Artifacts: 1, CacheSize: 200, Downloads: 5, DownloadedBytes: 1000}, - {Ecosystem: "gem", Packages: 3, Versions: 4}, - } - - totals, rows := analyticsView(stats) - - if totals.DownloadedBytes != 4000 { - t.Errorf("DownloadedBytes = %d, want 4000", totals.DownloadedBytes) - } - if totals.Downloads != "9" { - t.Errorf("Downloads = %q, want %q", totals.Downloads, "9") - } - if totals.CachedArtifacts != "3" { - t.Errorf("CachedArtifacts = %q, want %q", totals.CachedArtifacts, "3") - } - if totals.Packages != "6" { - t.Errorf("Packages = %q, want %q", totals.Packages, "6") - } - if totals.Versions != "7" { - t.Errorf("Versions = %q, want %q", totals.Versions, "7") - } - if totals.Ecosystems != 3 { - t.Errorf("Ecosystems = %d, want 3", totals.Ecosystems) - } - // gem has served nothing, so it is known but not active. - if totals.ActiveEcosystems != 2 { - t.Errorf("ActiveEcosystems = %d, want 2", totals.ActiveEcosystems) - } - // 4000 bytes served from 1700 bytes stored. - if totals.Amplification != "2.4x" { - t.Errorf("Amplification = %q, want %q", totals.Amplification, "2.4x") - } - - if len(rows) != 3 { - t.Fatalf("expected 3 rows, got %d", len(rows)) - } - - // Shares are of the grand total. - if rows[0].SharePct != "75.0" { - t.Errorf("npm share = %q, want 75.0", rows[0].SharePct) - } - if rows[1].SharePct != "25.0" { - t.Errorf("cargo share = %q, want 25.0", rows[1].SharePct) - } - // gem served nothing, so it contributes no share but still gets a row. - if rows[2].SharePct != "0.0" || rows[2].Ecosystem != "gem" { - t.Errorf("gem row = %+v, want a 0.0%% share", rows[2]) - } - if rows[0].AvgArtifactSize != "750 B" { - t.Errorf("npm AvgArtifactSize = %q, want %q", rows[0].AvgArtifactSize, "750 B") - } -} - -// With nothing cached at all the view must stay renderable rather than dividing -// by a zero total. -func TestAnalyticsViewEmpty(t *testing.T) { - totals, rows := analyticsView(nil) - - if len(rows) != 0 { - t.Errorf("expected no rows, got %+v", rows) - } - if totals.DownloadedBytes != 0 { - t.Errorf("DownloadedBytes = %d, want 0", totals.DownloadedBytes) - } - if totals.Downloaded != "0 B" { - t.Errorf("Downloaded = %q, want %q", totals.Downloaded, "0 B") - } - if totals.Amplification != "" { - t.Errorf("Amplification = %q, want empty", totals.Amplification) - } -} - -func TestAnalyticsPageRendersTotals(t *testing.T) { - tpl := &Templates{} - w := httptest.NewRecorder() - - totals, rows := analyticsView([]database.EcosystemStats{ - {Ecosystem: "npm", Packages: 1, Versions: 1, Artifacts: 1, CacheSize: 1000, Downloads: 3, DownloadedBytes: 3000}, - }) - data := AnalyticsData{Totals: totals, Ecosystems: rows} - data.Donut = donutView([]database.EcosystemStats{ - {Ecosystem: "npm", Packages: 1, Versions: 1, Artifacts: 1, CacheSize: 1000, Downloads: 3, DownloadedBytes: 3000}, - }, totals.DownloadedBytes, totals.Downloaded) - - if err := tpl.Render(w, "analytics", data); err != nil { - t.Fatalf("Render: %v", err) - } - - body := w.Body.String() - if strings.Contains(body, "ZgotmplZ") { - t.Error("a template value was sanitized away; check the donut dash geometry") - } - for _, want := range []string{ - "accumulated download size", - "2.9 KB", // the total, stated in the middle of the ring - "donut-slot-0", // the single slice takes the first categorical slot - "3.0x", // 3000 bytes served from 1000 stored - "/ui/analytics", // nav link renders on the page itself - } { - if !strings.Contains(body, want) { - t.Errorf("rendered page missing %q", want) - } - } -} - -// A retained snapshot must be labelled as one. The cache deliberately serves -// the last good rows when the query fails, so without this the page presents -// arbitrarily old figures as current and the only trace is a log line. -func TestAnalyticsPageFlagsStaleFigures(t *testing.T) { - tpl := &Templates{} - w := httptest.NewRecorder() - - totals, rows := analyticsView([]database.EcosystemStats{ - {Ecosystem: "npm", Artifacts: 1, CacheSize: 1000, Downloads: 3, DownloadedBytes: 3000}, - }) - data := AnalyticsData{ - Totals: totals, - Ecosystems: rows, - StatsFailed: true, - StatsStale: true, - StatsAge: "2 hours ago", - } - - if err := tpl.Render(w, "analytics", data); err != nil { - t.Fatalf("Render: %v", err) - } - - body := w.Body.String() - for _, want := range []string{"Figures may be out of date", "2 hours ago"} { - if !strings.Contains(body, want) { - t.Errorf("rendered page missing %q", want) - } - } - // The figures themselves still render — stale beats absent. - if !strings.Contains(body, "2.9 KB") { - t.Error("the retained snapshot was not rendered alongside the warning") - } -} - -// A failure with nothing retained keeps the existing unavailable message and -// must not also claim the figures below are stale, since there are none. -func TestAnalyticsPageStaleBannerNeedsASnapshot(t *testing.T) { - tpl := &Templates{} - w := httptest.NewRecorder() - - if err := tpl.Render(w, "analytics", AnalyticsData{StatsFailed: true}); err != nil { - t.Fatalf("Render: %v", err) - } - - body := w.Body.String() - if strings.Contains(body, "Figures may be out of date") { - t.Error("stale banner rendered with no retained snapshot") - } - if !strings.Contains(body, "the database query failed") { - t.Error("expected the unavailable message when the query failed with no snapshot") - } -} - -func TestAnalyticsPageRendersWithoutTraffic(t *testing.T) { - tpl := &Templates{} - w := httptest.NewRecorder() - - if err := tpl.Render(w, "analytics", AnalyticsData{}); err != nil { - t.Fatalf("Render: %v", err) - } - if body := w.Body.String(); !strings.Contains(body, "Nothing has been served from cache yet") { - t.Error("expected the empty-state message when no traffic has been recorded") - } -} - -func TestEcosystemMetricsMapping(t *testing.T) { - got := ecosystemMetrics([]database.EcosystemStats{ - {Ecosystem: "npm", Packages: 1, Versions: 2, Artifacts: 3, CacheSize: 4, Downloads: 5, DownloadedBytes: 6}, - }) - if len(got) != 1 { - t.Fatalf("expected 1 snapshot, got %d", len(got)) - } - m := got[0] - if m.Ecosystem != "npm" || m.Packages != 1 || m.Versions != 2 || m.Artifacts != 3 || - m.CacheSize != 4 || m.Downloads != 5 || m.DownloadedBytes != 6 { - t.Errorf("fields did not map across: %+v", m) - } -} - -// The metadata cache is stored apart from artifacts, so the page reports it in -// its own card rather than leaving it out of every figure. -func TestAnalyticsPageShowsMetadataCache(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - ts.server.cfg.CacheMetadata = true - - for _, e := range []database.MetadataCacheEntry{ - {Ecosystem: "pypi", Name: "requests", StoragePath: "_metadata/pypi/requests", Size: sql.NullInt64{Int64: 3 << 10, Valid: true}}, - {Ecosystem: "pypi", Name: "urllib3", StoragePath: "_metadata/pypi/urllib3", Size: sql.NullInt64{Int64: 1 << 10, Valid: true}}, - } { - if err := ts.db.UpsertMetadataCache(&e); err != nil { - t.Fatalf("UpsertMetadataCache: %v", err) - } - } - - body := ts.getOK(t, "/ui/analytics") - for _, want := range []string{"Metadata cache", "Cached documents", "4.0 KB", "100.0%"} { - if !strings.Contains(body, want) { - t.Errorf("rendered page missing %q", want) - } - } - if strings.Contains(body, "Metadata caching is off") { - t.Error("the off-state note rendered with cache_metadata on") - } -} - -func TestAnalyticsPageNotesMetadataCachingOff(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - - body := ts.getOK(t, "/ui/analytics") - for _, want := range []string{"Metadata cache", "Metadata caching is off", "No metadata cached yet"} { - if !strings.Contains(body, want) { - t.Errorf("rendered page missing %q", want) - } - } -} - -func TestMetadataCacheView(t *testing.T) { - v := metadataCacheView([]database.MetadataCacheStats{ - {Ecosystem: "npm", Entries: 3, Size: 3000}, - {Ecosystem: "pypi", Entries: 1, Size: 1000}, - }, true) - - if !v.Enabled || v.Entries != "4" || v.Size != "3.9 KB" { - t.Errorf("totals = %+v, want enabled, 4 entries, 3.9 KB", v) - } - if len(v.Rows) != 2 || v.Rows[0].SharePct != "75.0" || v.Rows[1].SharePct != "25.0" { - t.Errorf("rows = %+v, want npm 75.0%% then pypi 25.0%%", v.Rows) - } - - empty := metadataCacheView(nil, false) - if empty.Enabled || len(empty.Rows) != 0 || empty.Entries != "0" || empty.Size != "0 B" { - t.Errorf("empty view = %+v", empty) - } -} - -// The page reads the metadata card from the snapshot the gauges publish, so a -// page load issues no query of its own and the two always agree. -func TestAnalyticsPageMetadataCardUsesSnapshot(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - ts.server.cfg.CacheMetadata = true - - seed := func(name string, size int64) { - t.Helper() - if err := ts.db.UpsertMetadataCache(&database.MetadataCacheEntry{ - Ecosystem: "pypi", Name: name, StoragePath: "_metadata/pypi/" + name, - Size: sql.NullInt64{Int64: size, Valid: true}, - }); err != nil { - t.Fatalf("UpsertMetadataCache: %v", err) - } - } - - seed("requests", 1<<10) - ts.server.updateMetadataCacheStats() - if !strings.Contains(ts.getOK(t, "/ui/analytics"), "1.0 KB") { - t.Fatal("page did not show the snapshot the metrics tick read") - } - - seed("urllib3", 2<<10) - if body := ts.getOK(t, "/ui/analytics"); !strings.Contains(body, "1.0 KB") || strings.Contains(body, "3.0 KB") { - t.Error("a page load re-queried the database instead of serving the snapshot") - } - - ts.server.updateMetadataCacheStats() - if !strings.Contains(ts.getOK(t, "/ui/analytics"), "3.0 KB") { - t.Error("page did not pick up the refreshed snapshot") - } -} diff --git a/internal/server/api.go b/internal/server/api.go index 07080aa..ddb9ca7 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -1,12 +1,9 @@ package server import ( - "context" "encoding/json" - "errors" "net/http" "strings" - "time" shared "github.com/git-pkgs/enrichment" "github.com/git-pkgs/proxy/internal/database" @@ -23,20 +20,9 @@ const ( // APIHandler provides REST endpoints for package enrichment data. type APIHandler struct { - enrichment enricher + enrichment *enrichment.Service ecosystems *shared.EcosystemsClient db DBSearcher - // cooldown is nil with cooldown off. - cooldown *cooldownView -} - -// enricher is the part of enrichment.Service the API uses. -type enricher interface { - EnrichPackage(ctx context.Context, ecosystem, name string) (*enrichment.PackageInfo, error) - EnrichFull(ctx context.Context, ecosystem, name, version string) (*enrichment.EnrichmentResult, error) - CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]enrichment.VulnInfo, error) - GetLatestVersion(ctx context.Context, ecosystem, name string) (string, error) - BulkEnrichPackages(ctx context.Context, packages []struct{ Ecosystem, Name string }) map[string]*enrichment.PackageInfo } // DBSearcher defines the interface for database search operations. @@ -60,12 +46,6 @@ func NewAPIHandler(svc *enrichment.Service, db DBSearcher) *APIHandler { return h } -func (s *Server) newAPIHandler(svc *enrichment.Service) *APIHandler { - h := NewAPIHandler(svc, s.db) - h.cooldown = s.cooldown - return h -} - // PackageResponse contains enriched package metadata. type PackageResponse struct { Ecosystem string `json:"ecosystem"` @@ -77,9 +57,6 @@ type PackageResponse struct { Homepage string `json:"homepage,omitempty"` Repository string `json:"repository,omitempty"` RegistryURL string `json:"registry_url,omitempty"` - // Cooldown is the cooldown that applies to the package, in the - // configuration's syntax ("3d", "0"). - Cooldown string `json:"cooldown,omitempty"` } // VersionResponse contains enriched version metadata. @@ -89,15 +66,11 @@ type VersionResponse struct { Version string `json:"version"` License string `json:"license,omitempty"` PublishedAt string `json:"published_at,omitempty"` - // AvailableAt is when a version held back by cooldown leaves it. - AvailableAt string `json:"available_at,omitempty"` Integrity string `json:"integrity,omitempty"` Yanked bool `json:"yanked"` IsOutdated bool `json:"is_outdated"` } -const apiTimeFormat = "2006-01-02T15:04:05Z" - // VulnResponse contains vulnerability information. type VulnResponse struct { ID string `json:"id"` @@ -166,11 +139,12 @@ type BulkResponse struct { // Resolves namespaced package names (Composer vendor/name, npm @scope/name) from the path. func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - segments, err := packagePathSegments(r) - if err != nil { + wildcard := chi.URLParam(r, "*") + if err := validatePackagePath(wildcard); err != nil { badRequest(w, err.Error()) return } + segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) == 0 { badRequest(w, "ecosystem and name are required") @@ -201,12 +175,11 @@ func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) { Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(shared.CategorizeLicense(info.License)), + LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, RegistryURL: info.RegistryURL, - Cooldown: h.packageCooldown(ecosystem, fullName), } writeJSON(w, resp) return @@ -235,12 +208,11 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(shared.CategorizeLicense(info.License)), + LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, RegistryURL: info.RegistryURL, - Cooldown: h.packageCooldown(ecosystem, name), } writeJSON(w, resp) @@ -248,10 +220,6 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) { result, err := h.enrichment.EnrichFull(r.Context(), ecosystem, name, version) - if errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { - badRequest(w, "package identity is not supported for vulnerability lookups") - return - } if err != nil { writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to enrich version") return @@ -268,12 +236,11 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste Name: result.Package.Name, LatestVersion: result.Package.LatestVersion, License: result.Package.License, - LicenseCategory: string(shared.CategorizeLicense(result.Package.License)), + LicenseCategory: string(h.enrichment.CategorizeLicense(result.Package.License)), Description: result.Package.Description, Homepage: result.Package.Homepage, Repository: result.Package.Repository, RegistryURL: result.Package.RegistryURL, - Cooldown: h.packageCooldown(ecosystem, name), } } @@ -287,11 +254,8 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste Yanked: result.Version.Yanked, IsOutdated: result.IsOutdated, } - if published := result.Version.PublishedAt; !published.IsZero() { - resp.Version.PublishedAt = published.UTC().Format(apiTimeFormat) - if at := h.cooldown.availableAt(ecosystem, name, published, time.Now()); !at.IsZero() { - resp.Version.AvailableAt = at.UTC().Format(apiTimeFormat) - } + if !result.Version.PublishedAt.IsZero() { + resp.Version.PublishedAt = result.Version.PublishedAt.Format("2006-01-02T15:04:05Z") } } @@ -309,28 +273,16 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste writeJSON(w, resp) } -// packageCooldown returns the cooldown for a package in the configuration's -// syntax, or "" with cooldown off or not applied to the ecosystem. -func (h *APIHandler) packageCooldown(ecosystem, name string) string { - c := h.cooldown.forPackage(ecosystem, name) - switch { - case c == nil: - return "" - case c.Duration == cooldownOff: - return "0" - } - return c.Duration -} - // HandleVulnsPath dispatches /api/vulns/{ecosystem}/* to the vulns handler. // Supports both {name} and {name}/{version} paths with namespaced package names. func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - segments, err := packagePathSegments(r) - if err != nil { + wildcard := chi.URLParam(r, "*") + if err := validatePackagePath(wildcard); err != nil { badRequest(w, err.Error()) return } + segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) == 0 { badRequest(w, "ecosystem and name are required") @@ -353,10 +305,6 @@ func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { } vulns, err := h.enrichment.CheckVulnerabilities(r.Context(), ecosystem, name, version) - if errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { - badRequest(w, "package identity is not supported for vulnerability lookups") - return - } if err != nil { writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to check vulnerabilities") return @@ -420,7 +368,7 @@ func (h *APIHandler) HandleOutdated(w http.ResponseWriter, r *http.Request) { latest, err := h.enrichment.GetLatestVersion(r.Context(), pkg.Ecosystem, pkg.Name) if err == nil && latest != "" { result.LatestVersion = latest - result.IsOutdated = shared.IsOutdated(pkg.Version, latest) + result.IsOutdated = h.enrichment.IsOutdated(pkg.Version, latest) } resp.Results = append(resp.Results, result) @@ -467,7 +415,7 @@ func (h *APIHandler) HandleBulkLookup(w http.ResponseWriter, r *http.Request) { Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(shared.CategorizeLicense(info.License)), + LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, @@ -498,7 +446,7 @@ func (h *APIHandler) HandleBulkLookup(w http.ResponseWriter, r *http.Request) { Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(shared.CategorizeLicense(info.License)), + LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, @@ -689,7 +637,9 @@ func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request) licenseCategory := licenseCategoryUnknown if pkg.License.Valid { license = pkg.License.String - licenseCategory = string(shared.CategorizeLicense(license)) + if h.enrichment != nil { + licenseCategory = string(h.enrichment.CategorizeLicense(license)) + } } cachedAt := "" if pkg.CachedAt.Valid { diff --git a/internal/server/api_cooldown_test.go b/internal/server/api_cooldown_test.go deleted file mode 100644 index 4e3e3c7..0000000 --- a/internal/server/api_cooldown_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package server - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/enrichment" - "github.com/go-chi/chi/v5" -) - -func TestAPIVersionAvailableAt(t *testing.T) { - now := time.Now().UTC().Truncate(time.Second) - recent, old := now.Add(-time.Hour), now.Add(-96*time.Hour) - h := &APIHandler{ - enrichment: fakeEnricher{published: map[string]time.Time{"1.0.0": old, "2.0.0": recent}}, - cooldown: testCooldownView(t, config.CooldownConfig{Default: "3d"}), - } - r := chi.NewRouter() - r.Get("/api/package/{ecosystem}/*", h.HandlePackagePath) - - tests := []struct{ version, availableAt string }{ - {"2.0.0", recent.Add(72 * time.Hour).Format(apiTimeFormat)}, - {"1.0.0", ""}, - } - for _, tt := range tests { - t.Run(tt.version, func(t *testing.T) { - w := httptest.NewRecorder() - r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/package/npm/leftpad/"+tt.version, nil)) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, body = %s", w.Code, w.Body) - } - var resp EnrichmentResponse - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } - if resp.Package == nil || resp.Package.Cooldown != "3d" { - t.Errorf("package = %+v, want cooldown 3d", resp.Package) - } - if resp.Version == nil || resp.Version.AvailableAt != tt.availableAt { - t.Errorf("version = %+v, want available_at %q", resp.Version, tt.availableAt) - } - }) - } -} - -// fakeEnricher answers version lookups with the given publish times and -// knows no packages by name alone. -type fakeEnricher struct{ published map[string]time.Time } - -func (fakeEnricher) EnrichPackage(context.Context, string, string) (*enrichment.PackageInfo, error) { - return nil, nil -} - -func (f fakeEnricher) EnrichFull(_ context.Context, ecosystem, name, version string) (*enrichment.EnrichmentResult, error) { - return &enrichment.EnrichmentResult{ - Package: &enrichment.PackageInfo{Ecosystem: ecosystem, Name: name}, - Version: &enrichment.VersionInfo{Number: version, PublishedAt: f.published[version]}, - }, nil -} - -func (fakeEnricher) CheckVulnerabilities(context.Context, string, string, string) ([]enrichment.VulnInfo, error) { - return nil, nil -} - -func (fakeEnricher) GetLatestVersion(context.Context, string, string) (string, error) { - return "", nil -} - -func (fakeEnricher) BulkEnrichPackages(context.Context, []struct{ Ecosystem, Name string }) map[string]*enrichment.PackageInfo { - return nil -} diff --git a/internal/server/apk_snapshots.go b/internal/server/apk_snapshots.go deleted file mode 100644 index ee8e3c9..0000000 --- a/internal/server/apk_snapshots.go +++ /dev/null @@ -1,48 +0,0 @@ -package server - -import ( - "context" - "time" -) - -// apkSnapshotPruneInterval is how often snapshots kept past -// upstream.apk_snapshots are deleted. -const apkSnapshotPruneInterval = time.Hour - -// startAPKSnapshotPruning deletes APK index snapshots that were replaced more -// than upstream.apk_snapshots ago, once at startup and then every hour. Their -// blobs go through the pending-delete queue like evicted cache entries. -func (s *Server) startAPKSnapshotPruning(ctx context.Context) { - keep := s.cfg.ParseAPKSnapshots() - if keep <= 0 { - return - } - if !s.cfg.CacheMetadata { - s.logger.Warn("upstream.apk_snapshots needs cache_metadata, snapshots disabled") - return - } - - ticker := time.NewTicker(apkSnapshotPruneInterval) - defer ticker.Stop() - for { - s.pruneAPKSnapshots(ctx, keep, time.Now()) - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - } -} - -func (s *Server) pruneAPKSnapshots(ctx context.Context, keep time.Duration, now time.Time) { - s.runExclusive(ctx, "apk-snapshot-prune", func() { - n, err := s.db.PruneMetadataSnapshots("alpine", now.Add(-keep)) - if err != nil { - s.logger.Warn("apk snapshots: prune failed", "error", err) - return - } - if n > 0 { - s.logger.Info("apk snapshots: pruned", "deleted", n) - } - }) -} diff --git a/internal/server/apk_snapshots_test.go b/internal/server/apk_snapshots_test.go deleted file mode 100644 index 191e073..0000000 --- a/internal/server/apk_snapshots_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package server - -import ( - "context" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/handler" - "github.com/git-pkgs/proxy/internal/storage" - "github.com/git-pkgs/registries/fetch" - "github.com/go-chi/chi/v5" -) - -// TestAPKSnapshotsConfigured checks that upstream.apk_snapshots enables the -// dated APK paths, with the digest the file storage records for the cached -// index matching the copy kept as a snapshot. -func TestAPKSnapshotsConfigured(t *testing.T) { - const index = "signed index" - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, index) - })) - defer upstream.Close() - - db, err := database.Create(t.TempDir() + "/test.db") - if err != nil { - t.Fatalf("creating database: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) - if err != nil { - t.Fatalf("opening storage: %v", err) - } - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - proxy := handler.NewProxy(db, store, fetch.NewFetcher(), fetch.NewResolver(), logger) - proxy.HTTPClient = upstream.Client() - proxy.CacheMetadata = true - - cfg := config.Default() - cfg.BaseURL = "http://proxy.test" - cfg.Upstream.APK = map[string]string{"alpine": upstream.URL} - cfg.Upstream.APKSnapshots = "30d" - s := &Server{cfg: cfg, logger: logger} - r := chi.NewRouter() - s.mountProtocolHandlers(r, proxy) - - today := time.Now().UTC().Format("2006-01-02") - for _, target := range []string{ - "/apk/alpine/v3.22/main/x86_64/APKINDEX.tar.gz", - "/apk/alpine@" + today + "/v3.22/main/x86_64/APKINDEX.tar.gz", - } { - w := httptest.NewRecorder() - r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - if w.Code != http.StatusOK || w.Body.String() != index { - t.Fatalf("%s: status = %d, body = %q, want 200 %q", target, w.Code, w.Body.String(), index) - } - } -} diff --git a/internal/server/breakers.go b/internal/server/breakers.go deleted file mode 100644 index 383e0db..0000000 --- a/internal/server/breakers.go +++ /dev/null @@ -1,130 +0,0 @@ -package server - -import ( - "log/slog" - "sync" - - "github.com/git-pkgs/proxy/internal/metrics" -) - -// Gauge values for proxy_circuit_breaker_state. The fetcher reports only open -// or closed, so half-open (1) is never published. -const ( - breakerGaugeClosed = 0 - breakerGaugeOpen = 2 -) - -const ( - breakerStateOpen = "open" - breakerStateClosed = "closed" -) - -// breakerStateSource reports circuit breaker state per upstream registry, keyed -// by the identifier the fetcher derives from the fetch URL, with values -// breakerStateOpen or breakerStateClosed. Implemented by -// fetch.CircuitBreakerFetcher. -type breakerStateSource interface { - GetBreakerState() map[string]string -} - -// breakerMonitor mirrors the artifact fetcher's per-registry circuit breaker -// state into Prometheus metrics, the health report, and the log. -// -// Breaker state lives only in the fetcher's memory. While a breaker is open -// every artifact fetch it covers that misses the cache fails without reaching -// the upstream, bar the one probe per backoff interval the breaker admits to -// test recovery. That looks identical to an upstream outage from the outside: -// metadata still serves (it does not go through the fetcher), other registries -// still serve, and /health reports the database and storage as fine. Publishing -// the state makes that distinguishable. -type breakerMonitor struct { - source breakerStateSource - logger *slog.Logger - - // mu serializes snapshots. It guards seen, which holds one entry per - // registry that has tripped at least once in this process — the only - // registries published as metrics — and keeps each state read paired with - // the updates it produces. - mu sync.Mutex - seen map[string]string -} - -func newBreakerMonitor(source breakerStateSource, logger *slog.Logger) *breakerMonitor { - if logger == nil { - logger = slog.Default() - } - return &breakerMonitor{ - source: source, - logger: logger, - seen: map[string]string{}, - } -} - -// snapshot returns the current state of every breaker the fetcher has created, -// keyed by registry identifier, and mirrors it into the breaker metrics as a -// side effect. It returns nil for a nil monitor so callers that build a Server -// without a fetcher (tests) need no special case. -// -// The keys pass through unaltered into Prometheus labels and the /health body, -// neither of which is authenticated, so they are only as safe to publish as the -// fetcher makes them. It keys by the fetch URL's host, and where it cannot take -// a host from that URL — a signed composer dist.url that fails to parse, say — -// by an opaque keyed digest of the URL rather than the URL itself, so a -// credential carried in one does not reach either endpoint -// (github.com/git-pkgs/registries v0.9.0 and later). -// -// Only registries that have tripped at least once are published as metrics. -// The fetcher creates a breaker per identifier it fetches under, and for some -// ecosystems that identifier comes from upstream metadata rather than -// configuration (composer takes it from a package's dist.url, helm from the -// chart URLs in index.yaml), so publishing every one would let upstream content -// grow the series count for the life of the process — the more so for URLs with -// no host, which get an identifier apiece rather than sharing one. An -// identifier that has never tripped carries no information a series could -// convey; once it trips it keeps reporting, including the 0 that marks its -// recovery. /health is a per-request response rather than a persistent series, -// so it reports every breaker. -// -// Trips are counted on the closed→open transitions observed between calls, -// because the fetcher exposes current state rather than trip events: a breaker -// that opens and recovers entirely between two calls is not counted. -func (m *breakerMonitor) snapshot() map[string]string { - if m == nil || m.source == nil { - return nil - } - - m.mu.Lock() - defer m.mu.Unlock() - - // Read under the lock. Two concurrent snapshots — a /health request and a - // /metrics scrape landing during a transition — can otherwise apply their - // reads to seen in the opposite order, counting one trip twice, logging a - // close for a breaker that is still open, and leaving the gauge at 0 until - // the next call. - states := m.source.GetBreakerState() - - for registry, state := range states { - previous, published := m.seen[registry] - - switch { - case state == breakerStateOpen && previous != breakerStateOpen: - metrics.RecordCircuitBreakerTrip(registry) - m.logger.Error("circuit breaker open, artifact fetches for this registry "+ - "fail without contacting it", "registry", registry) - case state == breakerStateClosed && previous == breakerStateOpen: - m.logger.Info("circuit breaker closed", "registry", registry) - case state == breakerStateClosed && !published: - // Never tripped: nothing to publish. - continue - } - - gauge := breakerGaugeClosed - if state == breakerStateOpen { - gauge = breakerGaugeOpen - } - metrics.UpdateCircuitBreakerState(registry, gauge) - m.seen[registry] = state - } - - return states -} diff --git a/internal/server/breakers_test.go b/internal/server/breakers_test.go deleted file mode 100644 index 6cfb511..0000000 --- a/internal/server/breakers_test.go +++ /dev/null @@ -1,391 +0,0 @@ -package server - -import ( - "context" - "encoding/json" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" - "time" - - upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" - "github.com/git-pkgs/proxy/internal/metrics" - registryclient "github.com/git-pkgs/registries/client" - "github.com/git-pkgs/registries/fetch" - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" -) - -// cbThreshold in registries/fetch: failures inside the breaker's rolling window -// needed to trip it. The library counts the window, not a consecutive run, so -// these fetches only have to land close together, which they do. -const breakerTripFailures = 5 - -// fakeBreakerSource reports breaker state without a real fetcher, so tests can -// drive transitions that would otherwise need to wait out a 30s backoff. -type fakeBreakerSource struct { - states map[string]string -} - -func (f *fakeBreakerSource) GetBreakerState() map[string]string { - states := make(map[string]string, len(f.states)) - for registry, state := range f.states { - states[registry] = state - } - return states -} - -// newTrippedMonitor returns a monitor over a real circuit-breaker fetcher whose -// breaker for the test server's host has been tripped by repeated 5xx -// responses, plus that host. -func newTrippedMonitor(t *testing.T) (*breakerMonitor, string) { - t.Helper() - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusBadGateway) - })) - t.Cleanup(upstream.Close) - - monitor, fetcher, host := newMonitorFor(t, upstream) - for range breakerTripFailures { - if _, err := fetcher.Fetch(context.Background(), upstream.URL+"/artifact.tgz"); err == nil { - t.Fatal("fetch against a 502 upstream should fail") - } - } - return monitor, host -} - -// newMonitorFor builds a monitor over a circuit-breaker fetcher that talks to -// srv through its own client, bypassing the SSRF dial gate that would otherwise -// refuse the loopback address. -func newMonitorFor(t *testing.T, srv *httptest.Server) ( - monitor *breakerMonitor, fetcher *fetch.CircuitBreakerFetcher, host string, -) { - t.Helper() - - base := fetch.NewFetcher( - fetch.WithHTTPClient(srv.Client()), - fetch.WithMaxRetries(0), - ) - t.Cleanup(func() { _ = base.Close() }) - - fetcher = fetch.NewCircuitBreakerFetcher(base) - return newBreakerMonitor(fetcher, slog.New(slog.DiscardHandler)), - fetcher, - strings.TrimPrefix(srv.URL, "http://") -} - -// metricValue returns the value of the series carrying registry=want, and -// whether such a series exists at all. It collects rather than calling -// WithLabelValues, which would create the series it is looking for. -func metricValue(t *testing.T, collector prometheus.Collector, want string) (value float64, found bool) { - t.Helper() - - ch := make(chan prometheus.Metric, 64) - go func() { - collector.Collect(ch) - close(ch) - }() - - for metric := range ch { - var parsed dto.Metric - if err := metric.Write(&parsed); err != nil { - t.Fatalf("writing metric: %v", err) - } - for _, label := range parsed.GetLabel() { - if label.GetName() != "registry" || label.GetValue() != want { - continue - } - if gauge := parsed.GetGauge(); gauge != nil { - return gauge.GetValue(), true - } - return parsed.GetCounter().GetValue(), true - } - } - return 0, false -} - -// resetSeries drops any series for host left behind by an earlier test, since -// httptest ports can be reused within a process and the metrics registry is -// global. Absolute trip counts stay meaningful after it. -func resetSeries(host string) { - metrics.CircuitBreakerState.DeleteLabelValues(host) - metrics.CircuitBreakerTrips.DeleteLabelValues(host) -} - -func TestBreakerMonitor_OpenBreakerReportedAndCounted(t *testing.T) { - monitor, host := newTrippedMonitor(t) - resetSeries(host) - - if state := monitor.snapshot()[host]; state != breakerStateOpen { - t.Fatalf("state for %s = %q, want open", host, state) - } - - gauge, found := metricValue(t, metrics.CircuitBreakerState, host) - if !found { - t.Fatalf("no state gauge published for %s", host) - } - if gauge != breakerGaugeOpen { - t.Errorf("state gauge = %v, want %v", gauge, breakerGaugeOpen) - } - if trips, _ := metricValue(t, metrics.CircuitBreakerTrips, host); trips != 1 { - t.Errorf("trips = %v, want 1", trips) - } - - // A breaker that stays open is one trip, not one per scrape. - monitor.snapshot() - monitor.snapshot() - if trips, _ := metricValue(t, metrics.CircuitBreakerTrips, host); trips != 1 { - t.Errorf("trips after further snapshots = %v, want 1", trips) - } -} - -// A breaker per fetched host is created even for hosts that come from upstream -// metadata (composer dist.url, helm chart URLs), so publishing a series for -// every one of them would let upstream content grow the series count without -// bound. Only registries that have actually tripped are published. -func TestBreakerMonitor_HealthyRegistryPublishesNoSeries(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte("artifact")) - })) - defer upstream.Close() - - monitor, fetcher, host := newMonitorFor(t, upstream) - resetSeries(host) - - artifact, err := fetcher.Fetch(context.Background(), upstream.URL+"/artifact.tgz") - if err != nil { - t.Fatalf("fetch: %v", err) - } - _, _ = io.Copy(io.Discard, artifact.Body) - _ = artifact.Body.Close() - - // The breaker exists and is reported to /health... - if state := monitor.snapshot()[host]; state != breakerStateClosed { - t.Fatalf("state for %s = %q, want closed", host, state) - } - // ...but carries no metric series. - if _, found := metricValue(t, metrics.CircuitBreakerState, host); found { - t.Errorf("state gauge published for %s, want none until it trips", host) - } - if _, found := metricValue(t, metrics.CircuitBreakerTrips, host); found { - t.Errorf("trip counter published for %s, want none until it trips", host) - } -} - -// Once a registry has tripped it keeps reporting, so recovery is visible as a -// transition to 0 rather than as a series that disappears. -func TestBreakerMonitor_RecoveryReportedAfterTrip(t *testing.T) { - const host = "recovering.example.com" - - resetSeries(host) - - source := &fakeBreakerSource{states: map[string]string{host: breakerStateOpen}} - monitor := newBreakerMonitor(source, slog.New(slog.DiscardHandler)) - - monitor.snapshot() - if gauge, _ := metricValue(t, metrics.CircuitBreakerState, host); gauge != breakerGaugeOpen { - t.Fatalf("state gauge = %v, want %v", gauge, breakerGaugeOpen) - } - - source.states[host] = breakerStateClosed - if state := monitor.snapshot()[host]; state != breakerStateClosed { - t.Fatalf("state for %s = %q, want closed", host, state) - } - gauge, found := metricValue(t, metrics.CircuitBreakerState, host) - if !found { - t.Fatal("state gauge disappeared after recovery, want 0") - } - if gauge != breakerGaugeClosed { - t.Errorf("state gauge = %v, want %v", gauge, breakerGaugeClosed) - } - if trips, _ := metricValue(t, metrics.CircuitBreakerTrips, host); trips != 1 { - t.Errorf("trips = %v, want 1", trips) - } - - // Tripping again after recovery counts a second trip. - source.states[host] = breakerStateOpen - monitor.snapshot() - if trips, _ := metricValue(t, metrics.CircuitBreakerTrips, host); trips != 2 { - t.Errorf("trips after re-trip = %v, want 2", trips) - } -} - -func TestBreakerMonitor_NilSafe(t *testing.T) { - var nilMonitor *breakerMonitor - if states := nilMonitor.snapshot(); states != nil { - t.Errorf("nil monitor snapshot = %v, want nil", states) - } - if states := newBreakerMonitor(nil, nil).snapshot(); states != nil { - t.Errorf("snapshot without a state source = %v, want nil", states) - } -} - -func TestHealthEndpoint_ReportsOpenBreaker(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - - monitor, host := newTrippedMonitor(t) - ts.server.breakers = monitor - - req := httptest.NewRequest("GET", "/health", nil) - w := httptest.NewRecorder() - ts.handler.ServeHTTP(w, req) - - // An unreachable upstream is not this proxy being unhealthy: the breaker - // state is reported, but the probe still passes. - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) - } - - var resp HealthResponse - if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { - t.Fatalf("decoding response: %v", err) - } - if resp.Status != "ok" { - t.Errorf("status = %q, want ok", resp.Status) - } - if got := resp.CircuitBreakers[host]; got != breakerStateOpen { - t.Errorf("circuit_breakers[%s] = %q, want open", host, got) - } -} - -func TestHealthEndpoint_OmitsBreakersWhenNoneExist(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - - req := httptest.NewRequest("GET", "/health", nil) - w := httptest.NewRecorder() - ts.handler.ServeHTTP(w, req) - - if strings.Contains(w.Body.String(), "circuit_breakers") { - t.Errorf("body should omit circuit_breakers when no breaker exists: %s", w.Body.String()) - } -} - -// blockingBreakerSource holds each state read open until the test releases it, -// and reports every entry, so a read that escaped the monitor lock announces -// itself. inFlight is a second net: it catches an overlap anywhere in the test, -// not just while the release channel is held. -type blockingBreakerSource struct { - states map[string]string - entered chan struct{} - release chan struct{} - inFlight atomic.Int32 - overlap atomic.Bool -} - -func (b *blockingBreakerSource) GetBreakerState() map[string]string { - if b.inFlight.Add(1) > 1 { - b.overlap.Store(true) - } - defer b.inFlight.Add(-1) - - b.entered <- struct{}{} - <-b.release - - states := make(map[string]string, len(b.states)) - for registry, state := range b.states { - states[registry] = state - } - return states -} - -// /health requests and /metrics scrapes call snapshot concurrently. The state -// read has to happen under the same lock as the seen updates it feeds: read -// outside it, two snapshots straddling a transition can apply their reads to -// seen in the opposite order and count one trip twice. -func TestBreakerMonitor_SnapshotSerializesStateReads(t *testing.T) { - const host = "serialized.example.com" - - resetSeries(host) - - source := &blockingBreakerSource{ - states: map[string]string{host: breakerStateOpen}, - entered: make(chan struct{}, 2), - release: make(chan struct{}), - } - monitor := newBreakerMonitor(source, slog.New(slog.DiscardHandler)) - - first := make(chan map[string]string, 1) - go func() { first <- monitor.snapshot() }() - - // The first snapshot is inside GetBreakerState now, holding the lock. - <-source.entered - - second := make(chan map[string]string, 1) - go func() { second <- monitor.snapshot() }() - - // Nothing can release the first read, so the second cannot get past the - // lock: an unlocked read would have queued an entry on the channel. - select { - case <-source.entered: - t.Fatal("second snapshot read breaker state while the first still held the lock") - case <-time.After(100 * time.Millisecond): - } - - close(source.release) - - if state := (<-first)[host]; state != breakerStateOpen { - t.Errorf("first snapshot state for %s = %q, want open", host, state) - } - // Only reached once the first snapshot has returned the lock. - <-source.entered - if state := (<-second)[host]; state != breakerStateOpen { - t.Errorf("second snapshot state for %s = %q, want open", host, state) - } - - if source.overlap.Load() { - t.Error("two snapshots read breaker state concurrently") - } - // The same open breaker seen twice is one trip, which is what the paired - // read and update buys. - if trips, _ := metricValue(t, metrics.CircuitBreakerTrips, host); trips != 1 { - t.Errorf("trips = %v, want 1", trips) - } -} - -func TestHealthEndpoint_ReportsUpstreamHolds(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - - gate := registryclient.NewRateLimitGate(roundTripperFunc(func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: http.StatusTooManyRequests, - Header: http.Header{"Retry-After": []string{"600"}}, - Body: http.NoBody, - Request: req, - }, nil - })) - req, _ := http.NewRequest(http.MethodGet, "https://repo1.maven.org/maven2/x", nil) - if resp, err := gate.RoundTrip(req); err == nil { - _ = resp.Body.Close() - } - ts.server.rateLimits = upstreamhttp.NewSharedRateLimitGate(gate, nil, nil) - - w := httptest.NewRecorder() - ts.handler.ServeHTTP(w, httptest.NewRequest("GET", "/health", nil)) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) - } - - var resp HealthResponse - if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { - t.Fatalf("decoding response: %v", err) - } - until, err := time.Parse(time.RFC3339, resp.UpstreamHolds["repo1.maven.org"]) - if err != nil { - t.Fatalf("upstream_holds = %v, want repo1.maven.org with an RFC 3339 deadline", resp.UpstreamHolds) - } - if remaining := time.Until(until); remaining < 9*time.Minute || remaining > 10*time.Minute { - t.Errorf("hold ends in %s, want about 10m", remaining) - } -} - -type roundTripperFunc func(*http.Request) (*http.Response, error) - -func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { return fn(req) } diff --git a/internal/server/browse.go b/internal/server/browse.go index 1184f2e..504f5f1 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -1,93 +1,36 @@ package server import ( - "bufio" "encoding/json" - "errors" "fmt" "io" - "io/fs" "net/http" "path" "strings" - "sync" "github.com/git-pkgs/archives" "github.com/git-pkgs/archives/diff" - "github.com/git-pkgs/magic" "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/purl" "github.com/go-chi/chi/v5" ) -const ( - contentTypePlainText = "text/plain; charset=utf-8" - browseSniffSize = 512 - // browsePrefetchSize is how much of a browsed file is read before the - // response starts. Read errors within it still produce an error status. - browsePrefetchSize = 64 << 10 -) +const contentTypePlainText = "text/plain; charset=utf-8" -// maxBrowseArchiveSize caps the compressed artifact size the browse and diff -// endpoints will read. Artifacts larger than this are rejected to prevent +// maxBrowseArchiveSize caps how much data openArchive will buffer for +// prefix detection. Artifacts larger than this are rejected to prevent // memory exhaustion from a single request. const maxBrowseArchiveSize = 512 << 20 // 512 MB -// maxConcurrentCompares caps how many version diffs run at once. Each one -// buffers both archives whole, up to maxBrowseArchiveSize apiece, so without -// a cap parallel requests could exhaust memory. -const maxConcurrentCompares = 2 - -// compareRetryAfter is the Retry-After, in seconds, sent with a diff refused -// because maxConcurrentCompares are already running. -const compareRetryAfter = "5" - -// compareLimiter counts running version diffs. The zero value is ready to use. -type compareLimiter struct { - mu sync.Mutex - active int -} - -// tryAcquire takes a slot if fewer than maxConcurrentCompares are taken. -func (l *compareLimiter) tryAcquire() bool { - l.mu.Lock() - defer l.mu.Unlock() - if l.active >= maxConcurrentCompares { - return false +// archiveFilename returns a filename suitable for archive format detection. +// Some ecosystems (e.g. composer) store artifacts with bare hash filenames +// that have no extension. This adds .zip when the original has no extension +// and the content is likely a zip archive. +func archiveFilename(filename string) string { + if path.Ext(filename) == "" { + return filename + ".zip" } - l.active++ - return true -} - -func (l *compareLimiter) release() { - l.mu.Lock() - defer l.mu.Unlock() - l.active-- -} - -// firstBrowsableArtifact returns the first cached artifact that can be opened as -// an archive, or nil if the version has none. -// -// A version's artifact list is not all archives: a PEP 658 core-metadata sidecar -// resolves to the same name and version as the distribution it describes, so it -// is cached under that version too. Sidecars are plain text, and because '-' -// sorts before '.' one can even precede the real distribution in the -// filename-ordered list, so selecting blindly would hand openArchive a file it -// cannot parse. -func firstBrowsableArtifact(artifacts []database.Artifact) *database.Artifact { - for i := range artifacts { - if artifacts[i].StoragePath.Valid && !isMetadataSidecar(artifacts[i].Filename) { - return &artifacts[i] - } - } - - return nil -} - -// isMetadataSidecar reports whether filename is a core-metadata sidecar rather -// than a distribution archive. -func isMetadataSidecar(filename string) bool { - return strings.HasSuffix(filename, handler.PyPIMetadataSuffix) + return filename } // detectSingleRootDir returns the single top-level directory name if all files @@ -95,65 +38,57 @@ func isMetadataSidecar(filename string) bool { // "repo-hash/"). Returns "" if there's no single root or the archive is flat. func detectSingleRootDir(reader archives.Reader) string { files, err := reader.List() - if err != nil { + if err != nil || len(files) == 0 { return "" } - return archives.CommonRoot(files) + var root string + for _, f := range files { + parts := strings.SplitN(f.Path, "/", 2) //nolint:mnd // split into dir + rest + if len(parts) == 0 { + continue + } + dir := parts[0] + if root == "" { + root = dir + } else if dir != root { + return "" + } + } + + if root == "" { + return "" + } + return root + "/" } -// openArchive opens a cached artifact as a random-access archive reader, -// auto-detecting and stripping a single top-level directory prefix (like -// GitHub zipballs). For npm, the hardcoded "package/" prefix takes precedence. -// The whole artifact is buffered, so only the version diff uses it; listing -// and file reads use the streaming browser instead. +// openArchive opens a cached artifact as an archive reader, auto-detecting +// and stripping a single top-level directory prefix (like GitHub zipballs). +// For npm, the hardcoded "package/" prefix takes precedence. func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations - data, err := readBrowseInput(content, maxBrowseArchiveSize) + fname := archiveFilename(filename) + + limited := io.LimitReader(content, maxBrowseArchiveSize+1) + data, err := io.ReadAll(limited) if err != nil { - return nil, err + return nil, fmt.Errorf("reading artifact: %w", err) + } + if int64(len(data)) > maxBrowseArchiveSize { + return nil, fmt.Errorf("artifact too large for browsing (%d bytes)", len(data)) } if ecosystem == "npm" { - return archives.OpenBytesWithPrefix(filename, data, npmPackagePrefix) + return archives.OpenBytesWithPrefix(fname, data, "package/") } - probe, err := archives.OpenBytes(filename, data) + probe, err := archives.OpenBytes(fname, data) if err != nil { return nil, err } prefix := detectSingleRootDir(probe) _ = probe.Close() - return archives.OpenBytesWithPrefix(filename, data, prefix) -} - -// browseLimitsOrDefault returns the stream limits for listing and file reads. -func (s *Server) browseLimitsOrDefault() archives.StreamOptions { - if s.browseLimits != (archives.StreamOptions{}) { - return s.browseLimits - } - return defaultBrowseLimits -} - -// openBrowseArchive prepares a cached artifact for streaming. content is the -// already open storage reader; later passes reopen the artifact. -func (s *Server) openBrowseArchive(r *http.Request, artifact *database.Artifact, content io.Reader, ecosystem string) (*archives.Browser, error) { - storagePath := artifact.StoragePath.String - reopen := func() (io.ReadCloser, error) { - return s.storage.Open(r.Context(), storagePath) - } - return newBrowseArchive(artifact.Filename, ecosystem, content, reopen, s.browseLimitsOrDefault()) -} - -// browseArchiveError reports a failure to read an archive for browsing. -func (s *Server) browseArchiveError(w http.ResponseWriter, err error, filename string) { - if isBrowseLimitError(err) { - s.logger.Warn("archive exceeds browse limits", "error", err, "filename", filename) - internalError(w, "archive exceeds browse limits") - return - } - s.logger.Error("failed to open archive", "error", err, "filename", filename) - internalError(w, "failed to open archive") + return archives.OpenBytesWithPrefix(fname, data, prefix) } // BrowseListResponse contains the file listing for a directory in an archives. @@ -194,11 +129,12 @@ type BrowseFileInfo struct { // {name}/{version}/file/{path} -> browse file func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - segments, err := packagePathSegments(r) - if err != nil { + wildcard := chi.URLParam(r, "*") + if err := validatePackagePath(wildcard); err != nil { badRequest(w, err.Error()) return } + segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) < 2 { badRequest(w, "ecosystem, name, and version required") @@ -249,11 +185,12 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) { // Supported paths: {name}/{fromVersion}/{toVersion} func (s *Server) handleComparePath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - segments, err := packagePathSegments(r) - if err != nil { + wildcard := chi.URLParam(r, "*") + if err := validatePackagePath(wildcard); err != nil { badRequest(w, err.Error()) return } + segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) < 3 { badRequest(w, "ecosystem, name, fromVersion, and toVersion required") @@ -273,7 +210,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n dirPath := r.URL.Query().Get("path") // Get the artifact for this version - versionPURL := s.cachedVersionPURL(ecosystem, name, version) + versionPURL := purl.MakePURLString(ecosystem, name, version) artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL) if err != nil { notFound(w, "version not found") @@ -285,7 +222,14 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n return } - cachedArtifact := firstBrowsableArtifact(artifacts) + // Find the first cached artifact + var cachedArtifact *database.Artifact + for i := range artifacts { + if artifacts[i].StoragePath.Valid { + cachedArtifact = &artifacts[i] + break + } + } if cachedArtifact == nil { notFound(w, "artifact not cached") @@ -301,16 +245,20 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n } defer func() { _ = artifactReader.Close() }() - archive, err := s.openBrowseArchive(r, cachedArtifact, artifactReader, ecosystem) + // Open archive with auto-detected prefix stripping + archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem) if err != nil { - s.browseArchiveError(w, err, cachedArtifact.Filename) + s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename) + internalError(w, "failed to open archive") return } + defer func() { _ = archiveReader.Close() }() - // List files in the directory, with the root prefix stripped - files, err := archive.ListDir(dirPath) + // List files in the directory + files, err := archiveReader.ListDir(dirPath) if err != nil { - s.browseArchiveError(w, err, cachedArtifact.Filename) + s.logger.Error("failed to list directory", "error", err, "path", dirPath) + internalError(w, "failed to list directory") return } @@ -356,7 +304,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n } // Get the artifact for this version - versionPURL := s.cachedVersionPURL(ecosystem, name, version) + versionPURL := purl.MakePURLString(ecosystem, name, version) artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL) if err != nil { notFound(w, "version not found") @@ -368,7 +316,14 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n return } - cachedArtifact := firstBrowsableArtifact(artifacts) + // Find the first cached artifact + var cachedArtifact *database.Artifact + for i := range artifacts { + if artifacts[i].StoragePath.Valid { + cachedArtifact = &artifacts[i] + break + } + } if cachedArtifact == nil { notFound(w, "artifact not cached") @@ -384,185 +339,116 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n } defer func() { _ = artifactReader.Close() }() - archive, err := s.openBrowseArchive(r, cachedArtifact, artifactReader, ecosystem) + // Open archive with auto-detected prefix stripping + archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem) if err != nil { - s.browseArchiveError(w, err, cachedArtifact.Filename) + s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename) + internalError(w, "failed to open archive") return } + defer func() { _ = archiveReader.Close() }() - // Find the file and stream it straight from the archive - fileReader, err := archive.Extract(filePath) - switch { - case errors.Is(err, fs.ErrNotExist), errors.Is(err, archives.ErrIsDir): - notFound(w, "file not found") - return - case err != nil: - s.browseArchiveError(w, err, cachedArtifact.Filename) + // Extract the file + fileReader, err := archiveReader.Extract(filePath) + if err != nil { + if strings.Contains(err.Error(), "not found") { + notFound(w, "file not found") + return + } + s.logger.Error("failed to extract file", "error", err, "path", filePath) + internalError(w, "failed to extract file") return } defer func() { _ = fileReader.Close() }() - s.writeBrowseFile(w, fileReader, filePath, cachedArtifact.Filename) -} - -// writeBrowseFile sends a file streamed from an archive. The start of the file -// is read before any headers, so a truncated or over-limit entry that fails -// there still gets an error status. A failure after the response has started -// aborts it, so the client sees an incomplete download rather than a -// successful one. -func (s *Server) writeBrowseFile(w http.ResponseWriter, file io.Reader, filePath, artifactName string) { - source := &trackedReader{reader: file} - content := bufio.NewReaderSize(source, browsePrefetchSize) - head, err := content.Peek(browsePrefetchSize) - if err != nil && !errors.Is(err, io.EOF) { - s.browseReadError(w, err, filePath, artifactName) - return - } - - contentType, knownPath := detectContentTypeFromPath(filePath) - if !knownPath { - contentType = detectContentTypeFromPrefix(head[:min(len(head), browseSniffSize)]) - } + contentType := detectContentType(filePath) w.Header().Set("Content-Type", contentType) - w.Header().Set("Content-Security-Policy", "sandbox; "+frameAncestorsNone) + w.Header().Set("Content-Security-Policy", "sandbox") w.Header().Set("X-Content-Type-Options", "nosniff") _, filename := path.Split(filePath) w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", filename)) - written, err := io.Copy(w, content) - if err == nil || source.err == nil { - // A write error without a read error means the client went away. - return - } - s.logger.Error("failed to stream file from archive", "error", source.err, - "path", filePath, "filename", artifactName, "bytes", written) - // Headers are already committed: finishing normally would turn a truncated - // file into a seemingly successful download. Let net/http close the HTTP/1 - // connection or reset the HTTP/2 stream instead. - panic(http.ErrAbortHandler) + // Stream the file + _, _ = io.Copy(w, fileReader) } -// browseReadError reports a failure reading a file before its response starts. -func (s *Server) browseReadError(w http.ResponseWriter, err error, filePath, artifactName string) { - if isBrowseLimitError(err) { - s.logger.Warn("archive exceeds browse limits", "error", err, "path", filePath, "filename", artifactName) - internalError(w, "archive exceeds browse limits") - return - } - s.logger.Error("failed to read file from archive", "error", err, "path", filePath, "filename", artifactName) - internalError(w, "failed to read file") -} - -// trackedReader records the first read error other than io.EOF, so a failed -// copy can be told apart from a failed write to the client. -type trackedReader struct { - reader io.Reader - err error -} - -func (r *trackedReader) Read(p []byte) (int, error) { - n, err := r.reader.Read(p) - if err != nil && !errors.Is(err, io.EOF) && r.err == nil { - r.err = err - } - return n, err -} - -func detectContentTypeFromPath(filename string) (string, bool) { +// detectContentType returns an appropriate content type based on file extension. +func detectContentType(filename string) string { ext := strings.ToLower(path.Ext(filename)) switch ext { // Text formats case ".txt", ".md", ".markdown": - return contentTypePlainText, true + return contentTypePlainText case ".html", ".htm", ".xhtml": - return contentTypePlainText, true + return contentTypePlainText case ".css": - return "text/css; charset=utf-8", true + return "text/css; charset=utf-8" case ".js", ".mjs": - return "application/javascript; charset=utf-8", true + return "application/javascript; charset=utf-8" case ".json": - return "application/json; charset=utf-8", true + return "application/json; charset=utf-8" case ".xml": - return "application/xml; charset=utf-8", true + return "application/xml; charset=utf-8" case ".yaml", ".yml": - return "text/yaml; charset=utf-8", true + return "text/yaml; charset=utf-8" case ".toml": - return "text/toml; charset=utf-8", true + return "text/toml; charset=utf-8" // Programming languages case ".go": - return "text/x-go; charset=utf-8", true + return "text/x-go; charset=utf-8" case ".rs": - return "text/x-rust; charset=utf-8", true + return "text/x-rust; charset=utf-8" case ".py": - return "text/x-python; charset=utf-8", true + return "text/x-python; charset=utf-8" case ".rb": - return "text/x-ruby; charset=utf-8", true + return "text/x-ruby; charset=utf-8" case ".java": - return "text/x-java; charset=utf-8", true + return "text/x-java; charset=utf-8" case ".c", ".h": - return "text/x-c; charset=utf-8", true + return "text/x-c; charset=utf-8" case ".cpp", ".cc", ".cxx", ".hpp": - return "text/x-c++; charset=utf-8", true + return "text/x-c++; charset=utf-8" case ".ts": - return "text/typescript; charset=utf-8", true + return "text/typescript; charset=utf-8" case ".tsx": - return "text/tsx; charset=utf-8", true + return "text/tsx; charset=utf-8" case ".jsx": - return "text/jsx; charset=utf-8", true + return "text/jsx; charset=utf-8" case ".php": - return "text/x-php; charset=utf-8", true + return "text/x-php; charset=utf-8" // Config files case ".conf", ".config", ".ini": - return contentTypePlainText, true + return contentTypePlainText case ".sh", ".bash": - return "text/x-shellscript; charset=utf-8", true + return "text/x-shellscript; charset=utf-8" case ".dockerfile": - return "text/x-dockerfile; charset=utf-8", true + return "text/x-dockerfile; charset=utf-8" // Images case ".png": - return "image/png", true + return "image/png" case ".jpg", ".jpeg": - return "image/jpeg", true + return "image/jpeg" case ".gif": - return "image/gif", true + return "image/gif" case ".svg": - return contentTypePlainText, true + return contentTypePlainText case ".ico": - return "image/x-icon", true + return "image/x-icon" // Archives case ".zip", ".tar", ".gz", ".bz2", ".xz": - return "application/octet-stream", true + return "application/octet-stream" default: + // Try to detect if it looks like text if isLikelyText(filename) { - return contentTypePlainText, true + return contentTypePlainText } - return "", false - } -} - -func detectContentTypeFromPrefix(prefix []byte) string { - result := magic.DetectPrefix(prefix) - if result.Kind == magic.KindText { - return contentTypePlainText - } - - switch result.Format { - case "png": - return "image/png" - case "jpeg": - return "image/jpeg" - case "gif": - return "image/gif" - case "pdf": - return "application/pdf" - default: return "application/octet-stream" } } @@ -591,16 +477,11 @@ func isLikelyText(filename string) bool { } // BrowseSourceData contains data for the browse source page. -// -// Version is the decoded version, for display. EscapedVersion is the same value -// escaped as a single URL path segment and is what the links and the browse API -// calls must use; see database.Version.EscapedVersion. type BrowseSourceData struct { Layout - Ecosystem string - PackageName string - Version string - EscapedVersion string + Ecosystem string + PackageName string + Version string } // handleBrowseSource is now showBrowseSource in server.go, dispatched via handlePackagePath. @@ -618,12 +499,11 @@ type BrowseSourceData struct { // @Success 200 {object} map[string]any // @Failure 404 {object} ErrorResponse // @Failure 500 {object} ErrorResponse -// @Failure 503 {object} ErrorResponse // @Router /ui/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion} [get] func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, name, fromVersion, toVersion string) { // Get artifacts for both versions - fromPURL := s.cachedVersionPURL(ecosystem, name, fromVersion) - toPURL := s.cachedVersionPURL(ecosystem, name, toVersion) + fromPURL := purl.MakePURLString(ecosystem, name, fromVersion) + toPURL := purl.MakePURLString(ecosystem, name, toVersion) fromArtifacts, err := s.db.GetArtifactsByVersionPURL(fromPURL) if err != nil || len(fromArtifacts) == 0 { @@ -638,21 +518,25 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, } // Find cached artifacts - fromArtifact := firstBrowsableArtifact(fromArtifacts) - toArtifact := firstBrowsableArtifact(toArtifacts) + var fromArtifact, toArtifact *database.Artifact + for i := range fromArtifacts { + if fromArtifacts[i].StoragePath.Valid { + fromArtifact = &fromArtifacts[i] + break + } + } + for i := range toArtifacts { + if toArtifacts[i].StoragePath.Valid { + toArtifact = &toArtifacts[i] + break + } + } if fromArtifact == nil || toArtifact == nil { notFound(w, "one or both versions not cached") return } - if !s.compares.tryAcquire() { - w.Header().Set("Retry-After", compareRetryAfter) - writeError(w, http.StatusServiceUnavailable, ErrCodeBusy, "too many comparisons in progress, try again shortly") - return - } - defer s.compares.release() - // Open both archives fromReader, err := s.storage.Open(r.Context(), fromArtifact.StoragePath.String) if err != nil { @@ -699,17 +583,12 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, } // ComparePageData contains data for the version comparison page. -// -// FromVersion and ToVersion are decoded, for display; the Escaped variants are -// the path-segment form used to build the compare API URL. type ComparePageData struct { Layout - Ecosystem string - PackageName string - FromVersion string - ToVersion string - EscapedFromVersion string - EscapedToVersion string + Ecosystem string + PackageName string + FromVersion string + ToVersion string } // handleComparePage is now showComparePage in server.go, dispatched via handlePackagePath. diff --git a/internal/server/browse_archive.go b/internal/server/browse_archive.go deleted file mode 100644 index 4545ed7..0000000 --- a/internal/server/browse_archive.go +++ /dev/null @@ -1,60 +0,0 @@ -package server - -import ( - "errors" - "fmt" - "io" - - "github.com/git-pkgs/archives" -) - -// Limits applied when listing or reading a single file. Bodies are streamed -// rather than retained, so these bound decompression work per request; they -// match the limits the buffered archive reader enforces. -const ( - maxBrowseExpandedSize = 512 << 20 // 512 MB across all entries - maxBrowseEntrySize = 512 << 20 // 512 MB for any one entry - maxBrowseEntries = 100_000 -) - -// npmPackagePrefix is the directory npm wraps every tarball's contents in. -const npmPackagePrefix = "package/" - -var defaultBrowseLimits = archives.StreamOptions{ - MaxInputBytes: maxBrowseArchiveSize, - MaxEntryBytes: maxBrowseEntrySize, - MaxExpandedBytes: maxBrowseExpandedSize, - MaxEntries: maxBrowseEntries, -} - -var errArtifactTooLarge = errors.New("artifact too large for browsing") - -// isBrowseLimitError reports whether err comes from one of the browse size or -// entry-count limits rather than a malformed archive. -func isBrowseLimitError(err error) bool { - return errors.Is(err, errArtifactTooLarge) || - errors.Is(err, archives.ErrInputLimit) || - errors.Is(err, archives.ErrEntrySizeLimit) || - errors.Is(err, archives.ErrDecompressLimit) || - errors.Is(err, archives.ErrEntryLimit) -} - -// readBrowseInput reads all of r, failing once it exceeds limit bytes. -func readBrowseInput(r io.Reader, limit int64) ([]byte, error) { - data, err := io.ReadAll(io.LimitReader(r, limit+1)) - if err != nil { - return nil, fmt.Errorf("reading artifact: %w", err) - } - if int64(len(data)) > limit { - return nil, fmt.Errorf("%w (%d bytes)", errArtifactTooLarge, len(data)) - } - return data, nil -} - -func newBrowseArchive(filename, ecosystem string, content io.Reader, reopen func() (io.ReadCloser, error), limits archives.StreamOptions) (*archives.Browser, error) { - options := archives.BrowseOptions{Limits: limits, DetectRoot: true} - if ecosystem == "npm" { - options.StripPrefix = npmPackagePrefix - } - return archives.OpenBrowser(filename, content, reopen, options) -} diff --git a/internal/server/browse_archive_test.go b/internal/server/browse_archive_test.go deleted file mode 100644 index 838802e..0000000 --- a/internal/server/browse_archive_test.go +++ /dev/null @@ -1,771 +0,0 @@ -package server - -import ( - "archive/tar" - "archive/zip" - "bytes" - "compress/gzip" - "database/sql" - "encoding/json" - "fmt" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "net/url" - "os" - "path/filepath" - "runtime" - "strings" - "testing" - - "github.com/git-pkgs/archives" - "github.com/git-pkgs/proxy/internal/database" -) - -type archiveEntry struct { - name string - body string - dir bool -} - -func buildTar(t testing.TB, entries []archiveEntry) []byte { - t.Helper() - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - for _, e := range entries { - header := &tar.Header{Name: e.name, Mode: 0o644, Size: int64(len(e.body)), Typeflag: tar.TypeReg} - if e.dir { - header.Typeflag = tar.TypeDir - header.Mode = 0o755 - header.Size = 0 - } - if err := tw.WriteHeader(header); err != nil { - t.Fatalf("writing tar header: %v", err) - } - if _, err := io.WriteString(tw, e.body); err != nil { - t.Fatalf("writing tar body: %v", err) - } - } - if err := tw.Close(); err != nil { - t.Fatalf("closing tar: %v", err) - } - return buf.Bytes() -} - -func gzipBytes(t testing.TB, data []byte) []byte { - t.Helper() - buf := new(bytes.Buffer) - gw := gzip.NewWriter(buf) - if _, err := gw.Write(data); err != nil { - t.Fatalf("writing gzip: %v", err) - } - if err := gw.Close(); err != nil { - t.Fatalf("closing gzip: %v", err) - } - return buf.Bytes() -} - -func buildTarGz(t testing.TB, entries []archiveEntry) []byte { - t.Helper() - return gzipBytes(t, buildTar(t, entries)) -} - -func buildZip(t testing.TB, entries []archiveEntry) []byte { - t.Helper() - buf := new(bytes.Buffer) - zw := zip.NewWriter(buf) - for _, e := range entries { - f, err := zw.Create(e.name) - if err != nil { - t.Fatalf("creating zip entry: %v", err) - } - if _, err := io.WriteString(f, e.body); err != nil { - t.Fatalf("writing zip entry: %v", err) - } - } - if err := zw.Close(); err != nil { - t.Fatalf("closing zip: %v", err) - } - return buf.Bytes() -} - -// buildGem wraps entries in a gem: an outer tar holding data.tar.gz. -func buildGem(t testing.TB, entries []archiveEntry) []byte { - t.Helper() - return buildTar(t, []archiveEntry{ - {name: "metadata.gz", body: string(gzipBytes(t, []byte("--- {}\n")))}, - {name: "data.tar.gz", body: string(buildTarGz(t, entries))}, - }) -} - -// testBrowseVersion is the version every browse fixture is cached under. -const testBrowseVersion = "1.0.0" - -// cacheBrowseArtifact stores data as the cached artifact for ecosystem/name. -func cacheBrowseArtifact(t *testing.T, ts *testServer, ecosystem, name, filename string, data []byte) { - t.Helper() - - storagePath := fmt.Sprintf("%s-%s-%s", ecosystem, name, filename) - artifactsDir := filepath.Join(ts.tempDir, "artifacts") - if err := os.MkdirAll(artifactsDir, 0o755); err != nil { - t.Fatalf("creating artifacts dir: %v", err) - } - if err := os.WriteFile(filepath.Join(artifactsDir, storagePath), data, 0o644); err != nil { - t.Fatalf("writing artifact: %v", err) - } - - pkg := &database.Package{ - PURL: fmt.Sprintf("pkg:%s/%s", ecosystem, name), - Ecosystem: ecosystem, - Name: name, - } - if err := ts.db.UpsertPackage(pkg); err != nil { - t.Fatalf("upserting package: %v", err) - } - ver := &database.Version{PURL: pkg.PURL + "@" + testBrowseVersion, PackagePURL: pkg.PURL} - if err := ts.db.UpsertVersion(ver); err != nil { - t.Fatalf("upserting version: %v", err) - } - artifact := &database.Artifact{ - VersionPURL: ver.PURL, - Filename: filename, - UpstreamURL: "https://example.com/" + filename, - StoragePath: sql.NullString{String: storagePath, Valid: true}, - } - if err := ts.db.UpsertArtifact(artifact); err != nil { - t.Fatalf("upserting artifact: %v", err) - } -} - -func browseGet(ts *testServer, target string) *httptest.ResponseRecorder { - req := httptest.NewRequest(http.MethodGet, target, nil) - w := httptest.NewRecorder() - ts.handler.ServeHTTP(w, req) - return w -} - -func browseListURL(ecosystem, name, dir string) string { - return fmt.Sprintf("/ui/api/browse/%s/%s/%s?path=%s", ecosystem, name, testBrowseVersion, url.QueryEscape(dir)) -} - -func browseFileURL(ecosystem, name, file string) string { - return fmt.Sprintf("/ui/api/browse/%s/%s/%s/file/%s", ecosystem, name, testBrowseVersion, file) -} - -func decodeListing(t *testing.T, w *httptest.ResponseRecorder) BrowseListResponse { - t.Helper() - if w.Code != http.StatusOK { - t.Fatalf("listing status = %d, body %s", w.Code, w.Body.String()) - } - var resp BrowseListResponse - if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { - t.Fatalf("decoding listing: %v", err) - } - return resp -} - -func listingPaths(resp BrowseListResponse) []string { - paths := make([]string, len(resp.Files)) - for i, f := range resp.Files { - paths[i] = f.Path - } - return paths -} - -type browseFixture struct { - name string - ecosystem string - filename string - data []byte -} - -func browseFixtures(t *testing.T) []browseFixture { - t.Helper() - - rooted := []archiveEntry{ - {name: "repo-abc123/", dir: true}, - {name: "repo-abc123/README.md", body: "# rooted\n"}, - {name: "repo-abc123/src/", dir: true}, - {name: "repo-abc123/src/main.go", body: "package main\n"}, - {name: "repo-abc123/src/util/strings.go", body: "package util\n"}, - {name: "repo-abc123/docs/guide.md", body: "guide\n"}, - } - npm := []archiveEntry{ - {name: "package/package.json", body: `{"name":"x"}`}, - {name: "package/README.md", body: "# npm\n"}, - {name: "package/src/main.go", body: "package main\n"}, - {name: "package/src/util/strings.go", body: "package util\n"}, - {name: "other/ignored.txt", body: "outside the package prefix\n"}, - } - flat := []archiveEntry{ - {name: "README.md", body: "# flat\n"}, - {name: "src/main.go", body: "package main\n"}, - {name: "src/util/strings.go", body: "package util\n"}, - {name: "docs/guide.md", body: "guide\n"}, - } - - return []browseFixture{ - {"tar.gz with root dir", "cargo", "demo-1.0.0.crate", buildTarGz(t, rooted)}, - {"extensionless tar.gz", "cargo", "artifact", buildTarGz(t, rooted)}, - {"npm package prefix", "npm", "demo-1.0.0.tgz", buildTarGz(t, npm)}, - {"flat tar.gz", "cargo", "demo-1.0.0.tar.gz", buildTarGz(t, flat)}, - {"zip with root dir", "composer", "demo.zip", buildZip(t, rooted)}, - {"extensionless zip", "composer", "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", buildZip(t, rooted)}, - {"flat zip", "composer", "demo.zip", buildZip(t, flat)}, - {"gem", "gem", "demo-1.0.0.gem", buildGem(t, flat)}, - } -} - -func TestBrowseListingStripsPrefix(t *testing.T) { - for _, fx := range browseFixtures(t) { - t.Run(fx.name, func(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - cacheBrowseArtifact(t, ts, fx.ecosystem, "demo", fx.filename, fx.data) - - root := listingPaths(decodeListing(t, browseGet(ts, browseListURL(fx.ecosystem, "demo", "")))) - for _, p := range root { - if strings.HasPrefix(p, "repo-abc123/") || strings.HasPrefix(p, "package/") { - t.Errorf("root listing kept archive prefix: %q", p) - } - } - if !containsPath(root, "README.md") || !containsPath(root, "src/") { - t.Errorf("root listing = %v, want README.md and src/", root) - } - - src := listingPaths(decodeListing(t, browseGet(ts, browseListURL(fx.ecosystem, "demo", "src")))) - if !containsPath(src, "src/main.go") || !containsPath(src, "src/util/") { - t.Errorf("src listing = %v, want src/main.go and src/util/", src) - } - }) - } -} - -func containsPath(paths []string, want string) bool { - for _, p := range paths { - if p == want { - return true - } - } - return false -} - -// TestBrowseListingMatchesBufferedReader checks the streamed listing returns -// exactly what the buffered archive reader returned for the same request. -func TestBrowseListingMatchesBufferedReader(t *testing.T) { - dirs := []string{"", "/", "src", "src/", "/src", "src/util", "docs", "missing", "README.md"} - - for _, fx := range browseFixtures(t) { - t.Run(fx.name, func(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - cacheBrowseArtifact(t, ts, fx.ecosystem, "demo", fx.filename, fx.data) - - buffered, err := openArchive(fx.filename, bytes.NewReader(fx.data), fx.ecosystem) - if err != nil { - t.Fatalf("openArchive: %v", err) - } - defer func() { _ = buffered.Close() }() - - for _, dir := range dirs { - want, err := buffered.ListDir(dir) - if err != nil { - t.Fatalf("buffered ListDir(%q): %v", dir, err) - } - got := decodeListing(t, browseGet(ts, browseListURL(fx.ecosystem, "demo", dir))) - if len(got.Files) != len(want) { - t.Fatalf("ListDir(%q) = %v, want %d entries", dir, listingPaths(got), len(want)) - } - for i, f := range got.Files { - w := want[i] - if f.Path != w.Path || f.Name != w.Name || f.IsDir != w.IsDir || f.Size != w.Size { - t.Errorf("ListDir(%q)[%d] = %+v, want %+v", dir, i, f, w) - } - } - } - }) - } -} - -func TestBrowseFileContents(t *testing.T) { - for _, fx := range browseFixtures(t) { - t.Run(fx.name, func(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - cacheBrowseArtifact(t, ts, fx.ecosystem, "demo", fx.filename, fx.data) - - buffered, err := openArchive(fx.filename, bytes.NewReader(fx.data), fx.ecosystem) - if err != nil { - t.Fatalf("openArchive: %v", err) - } - defer func() { _ = buffered.Close() }() - - for _, file := range []string{"README.md", "src/main.go", "src/util/strings.go"} { - rc, err := buffered.Extract(file) - if err != nil { - t.Fatalf("buffered Extract(%q): %v", file, err) - } - want, _ := io.ReadAll(rc) - _ = rc.Close() - - w := browseGet(ts, browseFileURL(fx.ecosystem, "demo", file)) - if w.Code != http.StatusOK { - t.Fatalf("file %s status = %d, body %s", file, w.Code, w.Body.String()) - } - if w.Body.String() != string(want) { - t.Errorf("file %s = %q, want %q", file, w.Body.String(), want) - } - wantType, _ := detectContentTypeFromPath(file) - if got := w.Header().Get("Content-Type"); got != wantType { - t.Errorf("file %s Content-Type = %q, want %q", file, got, wantType) - } - if got := w.Header().Get("Content-Disposition"); !strings.Contains(got, filepath.Base(file)) { - t.Errorf("file %s Content-Disposition = %q", file, got) - } - } - - w := browseGet(ts, browseFileURL(fx.ecosystem, "demo", "missing.txt")) - if w.Code != http.StatusNotFound { - t.Errorf("missing file status = %d, want 404", w.Code) - } - }) - } -} - -func TestBrowseFileDirectoryNotFound(t *testing.T) { - for _, format := range []string{"tar.gz", "zip"} { - for _, explicit := range []bool{false, true} { - t.Run(fmt.Sprintf("%s/explicit=%t", format, explicit), func(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - var logs bytes.Buffer - ts.server.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelError})) - - entries := []archiveEntry{{name: "pkg/src/main.go", body: "package main\n"}} - if explicit { - entries = append([]archiveEntry{{name: "pkg/src/", dir: true}}, entries...) - } - data := buildTarGz(t, entries) - if format == "zip" { - data = buildZip(t, entries) - } - cacheBrowseArtifact(t, ts, "cargo", "demo", "demo."+format, data) - - for _, dir := range []string{"src", "src/"} { - response := browseGet(ts, browseFileURL("cargo", "demo", dir)) - if response.Code != http.StatusNotFound { - t.Errorf("directory %q returned %d, want 404: %s", dir, response.Code, response.Body.String()) - } - } - if logs.Len() != 0 { - t.Errorf("directory request logged an error: %s", logs.String()) - } - }) - } - } -} - -func TestBrowseFileFirstDuplicateWins(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - data := buildTarGz(t, []archiveEntry{ - {name: "package/index.js", body: "first"}, - {name: "package/index.js", body: "second"}, - }) - cacheBrowseArtifact(t, ts, "npm", "dup", "dup-1.0.0.tgz", data) - - w := browseGet(ts, browseFileURL("npm", "dup", "index.js")) - if w.Code != http.StatusOK || w.Body.String() != "first" { - t.Fatalf("got %d %q, want 200 \"first\"", w.Code, w.Body.String()) - } -} - -func TestBrowseSingleRootFile(t *testing.T) { - for _, format := range []string{"tar.gz", "zip"} { - t.Run(format, func(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - entries := []archiveEntry{{name: "README.md", body: "single file\n"}} - data := buildTarGz(t, entries) - if format == "zip" { - data = buildZip(t, entries) - } - filename := "single." + format - cacheBrowseArtifact(t, ts, "cargo", "single", filename, data) - paths := listingPaths(decodeListing(t, browseGet(ts, browseListURL("cargo", "single", "")))) - if len(paths) != 1 || paths[0] != "README.md" { - t.Fatalf("listing = %v, want README.md", paths) - } - response := browseGet(ts, browseFileURL("cargo", "single", "README.md")) - if response.Code != http.StatusOK || response.Body.String() != "single file\n" { - t.Fatalf("file = %d %q", response.Code, response.Body.String()) - } - r, err := openArchive(filename, bytes.NewReader(data), "cargo") - if err != nil { - t.Fatal(err) - } - defer func() { _ = r.Close() }() - files, err := r.List() - if err != nil || len(files) != 1 || files[0].Path != "README.md" { - t.Fatalf("diff reader listing = %+v, %v", files, err) - } - }) - } -} - -// TestBrowseFileEscapedPath requests files the way the source browser does, -// with each path segment percent-encoded, so names holding URL syntax resolve. -func TestBrowseFileEscapedPath(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - names := []string{"src/a#b.txt", "src/what?.txt", "src/100%.txt", "dir with space/c.txt"} - entries := make([]archiveEntry, len(names)) - for i, name := range names { - entries[i] = archiveEntry{name: "package/" + name, body: name} - } - cacheBrowseArtifact(t, ts, "npm", "escaped", "escaped-1.0.0.tgz", buildTarGz(t, entries)) - - for _, name := range names { - segments := strings.Split(name, "/") - for i, seg := range segments { - segments[i] = url.PathEscape(seg) - } - w := browseGet(ts, browseFileURL("npm", "escaped", strings.Join(segments, "/"))) - if w.Code != http.StatusOK || w.Body.String() != name { - t.Errorf("file %q: got %d %q, want 200 %q", name, w.Code, w.Body.String(), name) - } - } -} - -func TestBrowseLimits(t *testing.T) { - entries := []archiveEntry{ - {name: "package/a.txt", body: strings.Repeat("a", 100)}, - {name: "package/b.txt", body: strings.Repeat("b", 100)}, - {name: "package/c.txt", body: strings.Repeat("c", 100)}, - } - tarGz := buildTarGz(t, entries) - zipped := buildZip(t, entries) - - tests := []struct { - name string - filename string - data []byte - limits archives.StreamOptions - file string - }{ - {"entry count", "limit.tgz", tarGz, archives.StreamOptions{MaxEntries: 2}, "c.txt"}, - {"entry size", "limit.tgz", tarGz, archives.StreamOptions{MaxEntryBytes: 50}, "a.txt"}, - {"expanded size", "limit.tgz", tarGz, archives.StreamOptions{MaxExpandedBytes: 250}, "c.txt"}, - {"input size", "limit.tgz", tarGz, archives.StreamOptions{MaxInputBytes: int64(len(tarGz) / 2)}, "c.txt"}, - {"zip entry count", "limit.zip", zipped, archives.StreamOptions{MaxEntries: 2}, "c.txt"}, - {"zip input size", "limit.zip", zipped, archives.StreamOptions{MaxInputBytes: int64(len(zipped) / 2)}, "a.txt"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - ts.server.browseLimits = tt.limits - cacheBrowseArtifact(t, ts, "npm", "limited", tt.filename, tt.data) - - for _, target := range []string{ - browseListURL("npm", "limited", ""), - browseFileURL("npm", "limited", tt.file), - } { - w := browseGet(ts, target) - if w.Code != http.StatusInternalServerError { - t.Fatalf("%s status = %d, want 500 (body %s)", target, w.Code, w.Body.String()) - } - if !strings.Contains(w.Body.String(), "archive exceeds browse limits") { - t.Errorf("%s body = %s, want limit error", target, w.Body.String()) - } - } - }) - } -} - -// largeBrowseArchive is a gzip tarball with many unrelated entries and one -// small file. The padding compresses well, so the fixture is small on disk -// while its expanded size dwarfs what a streamed request should allocate. -func largeBrowseArchive(t testing.TB, entries, entrySize int) []byte { - t.Helper() - buf := new(bytes.Buffer) - gw := gzip.NewWriter(buf) - tw := tar.NewWriter(gw) - padding := make([]byte, entrySize) - write := func(name string, body []byte) { - if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil { - t.Fatalf("writing tar header: %v", err) - } - if _, err := tw.Write(body); err != nil { - t.Fatalf("writing tar body: %v", err) - } - } - write("repo-large/README.md", []byte("small file\n")) - for i := range entries { - write(fmt.Sprintf("repo-large/vendor/blob%05d.bin", i), padding) - } - if err := tw.Close(); err != nil { - t.Fatalf("closing tar: %v", err) - } - if err := gw.Close(); err != nil { - t.Fatalf("closing gzip: %v", err) - } - return buf.Bytes() -} - -// allocatedBytes returns the bytes allocated while fn runs. Total allocation -// bounds the peak heap growth fn can cause. -func allocatedBytes(fn func()) uint64 { - runtime.GC() - var before, after runtime.MemStats - runtime.ReadMemStats(&before) - fn() - runtime.ReadMemStats(&after) - return after.TotalAlloc - before.TotalAlloc -} - -func TestBrowseLargeArchiveMemory(t *testing.T) { - const ( - entries = 1024 - entrySize = 32 << 10 - expanded = entries * entrySize // 32 MiB - ) - data := largeBrowseArchive(t, entries, entrySize) - - ts := newTestServer(t) - defer ts.close() - cacheBrowseArtifact(t, ts, "cargo", "large", "large-1.0.0.crate", data) - - buffered := allocatedBytes(func() { - r, err := openArchive("large-1.0.0.crate", bytes.NewReader(data), "cargo") - if err != nil { - t.Fatalf("openArchive: %v", err) - } - _ = r.Close() - }) - - var listing BrowseListResponse - list := allocatedBytes(func() { - listing = decodeListing(t, browseGet(ts, browseListURL("cargo", "large", ""))) - }) - if !containsPath(listingPaths(listing), "README.md") || !containsPath(listingPaths(listing), "vendor/") { - t.Fatalf("listing = %v, want README.md and vendor/", listingPaths(listing)) - } - - var w *httptest.ResponseRecorder - file := allocatedBytes(func() { - w = browseGet(ts, browseFileURL("cargo", "large", "README.md")) - }) - if w.Code != http.StatusOK || w.Body.String() != "small file\n" { - t.Fatalf("file = %d %q, want 200 \"small file\\n\"", w.Code, w.Body.String()) - } - - t.Logf("compressed %d bytes, expanded %d bytes", len(data), expanded) - t.Logf("allocated: buffered open %d, streamed listing %d, streamed file %d", buffered, list, file) - - if buffered < expanded { - t.Fatalf("buffered open allocated %d bytes, expected at least the expanded %d", buffered, expanded) - } - limit := uint64(expanded / 4) - if list > limit { - t.Errorf("streamed listing allocated %d bytes, want under %d", list, limit) - } - if file > limit { - t.Errorf("streamed file read allocated %d bytes, want under %d", file, limit) - } -} - -func BenchmarkBrowseLargeArchive(b *testing.B) { - data := largeBrowseArchive(b, 1024, 32<<10) - artifactPath := filepath.Join(b.TempDir(), "large.crate") - if err := os.WriteFile(artifactPath, data, 0o644); err != nil { - b.Fatal(err) - } - - b.Run("buffered-list", func(b *testing.B) { benchBufferedBrowse(b, data, false) }) - b.Run("streamed-list", func(b *testing.B) { benchStreamedBrowse(b, artifactPath, false) }) - b.Run("buffered-file", func(b *testing.B) { benchBufferedBrowse(b, data, true) }) - b.Run("streamed-file", func(b *testing.B) { benchStreamedBrowse(b, artifactPath, true) }) -} - -// benchBufferedBrowse lists the root, or reads README.md, through openArchive. -func benchBufferedBrowse(b *testing.B, data []byte, readFile bool) { - b.ReportAllocs() - for b.Loop() { - r, err := openArchive("large.crate", bytes.NewReader(data), "cargo") - if err != nil { - b.Fatal(err) - } - if readFile { - drainBrowseFile(b, r.Extract) - } else if _, err := r.ListDir(""); err != nil { - b.Fatal(err) - } - _ = r.Close() - } -} - -// benchStreamedBrowse lists the root, or reads README.md, through browseArchive. -func benchStreamedBrowse(b *testing.B, artifactPath string, readFile bool) { - reopen := func() (io.ReadCloser, error) { return os.Open(artifactPath) } - b.ReportAllocs() - for b.Loop() { - content, err := reopen() - if err != nil { - b.Fatal(err) - } - a, err := newBrowseArchive("large.crate", "cargo", content, reopen, defaultBrowseLimits) - if err != nil { - b.Fatal(err) - } - if readFile { - drainBrowseFile(b, a.Extract) - } else if _, err := a.ListDir(""); err != nil { - b.Fatal(err) - } - _ = content.Close() - } -} - -func drainBrowseFile(b *testing.B, extract func(string) (io.ReadCloser, error)) { - b.Helper() - rc, err := extract("README.md") - if err != nil { - b.Fatal(err) - } - _, _ = io.Copy(io.Discard, rc) - _ = rc.Close() -} - -// truncatedTar is a TAR whose only entry declares size bytes but carries just -// body, ending without the rest of the entry or the end-of-archive marker. -func truncatedTar(t testing.TB, name string, size int64, body []byte) []byte { - t.Helper() - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: size}); err != nil { - t.Fatalf("writing tar header: %v", err) - } - if _, err := tw.Write(body); err != nil { - t.Fatalf("writing tar body: %v", err) - } - // Deliberately not closed: tw.Close would fail on the short entry. - return buf.Bytes() -} - -func TestBrowseFileReadErrorBeforeResponse(t *testing.T) { - complete := buildTar(t, []archiveEntry{{name: "package/data.txt", body: strings.Repeat("x", 4096)}}) - - tests := []struct { - name string - filename string - data []byte - limits archives.StreamOptions - wantBody string - }{ - { - name: "truncated entry body", - filename: "short-1.0.0.tgz", - data: gzipBytes(t, truncatedTar(t, "package/data.txt", 4096, []byte("abc"))), - wantBody: "failed to read file", - }, - { - name: "input limit inside entry", - filename: "short-1.0.0.tar", - data: complete, - limits: archives.StreamOptions{MaxInputBytes: 1024}, - wantBody: "archive exceeds browse limits", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - ts.server.browseLimits = tt.limits - cacheBrowseArtifact(t, ts, "npm", "short", tt.filename, tt.data) - - w := browseGet(ts, browseFileURL("npm", "short", "data.txt")) - if w.Code != http.StatusInternalServerError { - t.Fatalf("status = %d, want 500 (body %q)", w.Code, w.Body.String()) - } - if strings.Contains(w.Body.String(), "abc") || strings.Contains(w.Body.String(), "xxx") { - t.Errorf("error response leaked partial file content: %q", w.Body.String()) - } - if !strings.Contains(w.Body.String(), tt.wantBody) { - t.Errorf("body = %q, want %q", w.Body.String(), tt.wantBody) - } - }) - } -} - -func TestBrowseFileReadErrorAfterResponseStarts(t *testing.T) { - const size = 4 * browsePrefetchSize - body := bytes.Repeat([]byte("0123456789abcdef"), size/16) - complete := buildTar(t, []archiveEntry{{name: "package/big.txt", body: string(body)}}) - - tests := []struct { - name string - filename string - data []byte - limits archives.StreamOptions - wantErr bool - }{ - { - name: "complete file", - filename: "big-1.0.0.tar", - data: complete, - }, - { - name: "truncated entry body", - filename: "big-1.0.0.tgz", - data: gzipBytes(t, truncatedTar(t, "package/big.txt", size, body[:size/2])), - wantErr: true, - }, - { - name: "input limit inside entry", - filename: "big-1.0.0.tar", - data: complete, - limits: archives.StreamOptions{MaxInputBytes: size / 2}, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - ts.server.browseLimits = tt.limits - cacheBrowseArtifact(t, ts, "npm", "big", tt.filename, tt.data) - - srv := httptest.NewServer(ts.handler) - defer srv.Close() - - resp, err := http.Get(srv.URL + browseFileURL("npm", "big", "big.txt")) - if err != nil { - if tt.wantErr { - return - } - t.Fatalf("GET: %v", err) - } - defer func() { _ = resp.Body.Close() }() - got, readErr := io.ReadAll(resp.Body) - - if !tt.wantErr { - if resp.StatusCode != http.StatusOK || readErr != nil || !bytes.Equal(got, body) { - t.Fatalf("status %d, read error %v, %d of %d bytes", resp.StatusCode, readErr, len(got), len(body)) - } - return - } - if readErr == nil { - t.Fatalf("status %d with %d of %d bytes and no read error; want an aborted response", - resp.StatusCode, len(got), len(body)) - } - if len(got) >= len(body) { - t.Errorf("read %d bytes, want fewer than %d", len(got), len(body)) - } - }) - } -} diff --git a/internal/server/browse_bench_test.go b/internal/server/browse_bench_test.go index 840bc75..03f3f02 100644 --- a/internal/server/browse_bench_test.go +++ b/internal/server/browse_bench_test.go @@ -55,35 +55,3 @@ func BenchmarkOpenArchive(b *testing.B) { }) } } - -func BenchmarkDetectContentType(b *testing.B) { - cases := []struct { - name string - filename string - prefix []byte - knownPath bool - }{ - {"known-path", "README.md", nil, true}, - {"text-prefix", "artifact", bytes.Repeat([]byte("a"), browseSniffSize), false}, - {"png-prefix", "artifact", append([]byte("\x89PNG\r\n\x1a\n"), make([]byte, browseSniffSize-8)...), false}, - } - - for _, tc := range cases { - b.Run(tc.name, func(b *testing.B) { - b.ReportAllocs() - var contentType string - if tc.knownPath { - for b.Loop() { - contentType, _ = detectContentTypeFromPath(tc.filename) - } - } else { - for b.Loop() { - contentType = detectContentTypeFromPrefix(tc.prefix) - } - } - if contentType == "" { - b.Fatal("empty content type") - } - }) - } -} diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go index 34c864c..f1fb993 100644 --- a/internal/server/browse_test.go +++ b/internal/server/browse_test.go @@ -137,47 +137,29 @@ func TestHandleBrowseFile(t *testing.T) { t.Fatalf("failed to upsert artifact: %v", err) } - files := []struct { - path string - content string - contentType string - }{ - {"README.md", "# Test Package\n", contentTypePlainText}, - {"notes.data", "short text\n", contentTypePlainText}, - {"logo", "\x89PNG\r\n\x1a\nimage data", "image/png"}, - {"page", "", contentTypePlainText}, - {"misleading.txt", "\x89PNG\r\n\x1a\nimage data", contentTypePlainText}, - } - for _, file := range files { - t.Run(file.path, func(t *testing.T) { - req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/"+file.path, nil) - w := httptest.NewRecorder() - ts.handler.ServeHTTP(w, req) + // Test fetching a file + req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/README.md", nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) - } - if w.Body.String() != file.content { - t.Errorf("unexpected file content: %q", w.Body.String()) - } - if got := w.Header().Get("Content-Type"); got != file.contentType { - t.Errorf("Content-Type = %q, want %q", got, file.contentType) - } - if got, want := w.Header().Get("Content-Security-Policy"), "sandbox; frame-ancestors 'none'"; got != want { - t.Errorf("Content-Security-Policy = %q, want %q", got, want) - } - if got := w.Header().Get("X-Frame-Options"); got != "DENY" { - t.Errorf("X-Frame-Options = %q, want DENY", got) - } - if got := w.Header().Get("X-Content-Type-Options"); got != "nosniff" { - t.Errorf("X-Content-Type-Options = %q, want nosniff", got) - } - }) + if w.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) + } + + body := w.Body.String() + if body != "# Test Package\n" { + t.Errorf("unexpected file content: %q", body) + } + + // Check content type + contentType := w.Header().Get("Content-Type") + if contentType != contentTypePlainText { + t.Errorf("expected text/plain content type, got %q", contentType) } // Test fetching non-existent file - req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil) - w := httptest.NewRecorder() + req = httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil) + w = httptest.NewRecorder() ts.handler.ServeHTTP(w, req) if w.Code != http.StatusNotFound { @@ -185,52 +167,36 @@ func TestHandleBrowseFile(t *testing.T) { } } -func TestBrowseContentTypePolicy(t *testing.T) { +func TestDetectContentType(t *testing.T) { tests := []struct { - name string filename string - prefix []byte expectedCT string }{ - {"text extension", "file.txt", nil, contentTypePlainText}, - {"markdown extension", "file.md", nil, contentTypePlainText}, - {"JSON extension", "file.json", nil, "application/json; charset=utf-8"}, - {"JavaScript extension", "file.js", nil, "application/javascript; charset=utf-8"}, - {"Go extension", "file.go", nil, "text/x-go; charset=utf-8"}, - {"Python extension", "file.py", nil, "text/x-python; charset=utf-8"}, - {"Rust extension", "file.rs", nil, "text/x-rust; charset=utf-8"}, - {"HTML extension", "file.html", nil, contentTypePlainText}, - {"HTM extension", "file.htm", nil, contentTypePlainText}, - {"XHTML extension", "file.xhtml", nil, contentTypePlainText}, - {"SVG extension", "file.svg", nil, contentTypePlainText}, - {"PNG extension", "file.png", nil, "image/png"}, - {"JPEG extension", "file.jpg", nil, "image/jpeg"}, - {"README", "README", nil, contentTypePlainText}, - {"LICENSE", "LICENSE", nil, contentTypePlainText}, - {"Makefile", "Makefile", nil, contentTypePlainText}, - {"gitignore", ".gitignore", nil, contentTypePlainText}, - {"unknown empty", "file.bin", nil, "application/octet-stream"}, - {"extensionless PNG", "asset", []byte("\x89PNG\r\n\x1a\n"), "image/png"}, - {"extensionless JPEG", "asset", []byte("\xff\xd8\xff"), "image/jpeg"}, - {"extensionless GIF", "asset", []byte("GIF89a"), "image/gif"}, - {"extensionless PDF", "asset", []byte("%PDF-1.7"), "application/pdf"}, - {"extensionless text", "asset", []byte("plain text\n"), contentTypePlainText}, - {"extensionless HTML", "asset", []byte(""), contentTypePlainText}, - {"extensionless XML", "asset", []byte(""), contentTypePlainText}, - {"extensionless SVG", "asset", []byte(""), contentTypePlainText}, - {"extensionless ZIP", "asset", []byte("PK\x03\x04"), "application/octet-stream"}, - {"extensionless binary", "asset", []byte{0, 1, 2}, "application/octet-stream"}, - {"known path wins", "file.txt", []byte("\x89PNG\r\n\x1a\n"), contentTypePlainText}, + {"file.txt", contentTypePlainText}, + {"file.md", contentTypePlainText}, + {"file.json", "application/json; charset=utf-8"}, + {"file.js", "application/javascript; charset=utf-8"}, + {"file.go", "text/x-go; charset=utf-8"}, + {"file.py", "text/x-python; charset=utf-8"}, + {"file.rs", "text/x-rust; charset=utf-8"}, + {"file.html", contentTypePlainText}, + {"file.htm", contentTypePlainText}, + {"file.xhtml", contentTypePlainText}, + {"file.svg", contentTypePlainText}, + {"file.png", "image/png"}, + {"file.jpg", "image/jpeg"}, + {"README", contentTypePlainText}, + {"LICENSE", contentTypePlainText}, + {"Makefile", contentTypePlainText}, + {".gitignore", contentTypePlainText}, + {"file.bin", "application/octet-stream"}, } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, knownPath := detectContentTypeFromPath(tt.filename) - if !knownPath { - got = detectContentTypeFromPrefix(tt.prefix) - } + t.Run(tt.filename, func(t *testing.T) { + got := detectContentType(tt.filename) if got != tt.expectedCT { - t.Errorf("content type for %q with prefix %q = %q, want %q", tt.filename, tt.prefix, got, tt.expectedCT) + t.Errorf("detectContentType(%q) = %q, want %q", tt.filename, got, tt.expectedCT) } }) } @@ -289,10 +255,6 @@ func createTestArchive(t *testing.T) []byte { "package/lib/index.js": "module.exports = {};", "package/lib/helper.js": "module.exports.help = () => {};", "package/test/index.test.js": "// tests", - "package/notes.data": "short text\n", - "package/logo": "\x89PNG\r\n\x1a\nimage data", - "package/page": "", - "package/misleading.txt": "\x89PNG\r\n\x1a\nimage data", } for path, content := range files { @@ -433,10 +395,6 @@ func TestHandleBrowseSourcePage(t *testing.T) { } } - if !strings.Contains(body, "proxy test-version (test-commit)") { - t.Error("browse source footer should contain proxy build information, not the package version") - } - // Check that the escapeHTML function is present for XSS protection if !strings.Contains(body, "function escapeHTML(str)") { t.Error("browse source page missing escapeHTML function for XSS protection") @@ -457,10 +415,8 @@ func TestHandleBrowseSourcePage(t *testing.T) { if !strings.Contains(body, "const packageName = 'test-browse'") { t.Error("browse source page missing packageName variable") } - // The version reaches the browse API as one path segment, so the page holds - // its escaped form. - if !strings.Contains(body, "const versionPath = '1.0.0'") { - t.Error("browse source page missing versionPath variable") + if !strings.Contains(body, "const version = '1.0.0'") { + t.Error("browse source page missing version variable") } // Verify content type @@ -474,47 +430,6 @@ func TestHandleCompareDiff(t *testing.T) { ts := newTestServer(t) defer ts.close() - seedCompareVersions(t, ts) - - // Test the compare endpoint - req := httptest.NewRequest("GET", "/ui/api/compare/npm/test-compare/1.0.0/2.0.0", nil) - w := httptest.NewRecorder() - ts.handler.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) - } - - // Parse response - var result map[string]interface{} - if err := json.NewDecoder(w.Body).Decode(&result); err != nil { - t.Fatalf("failed to decode response: %v", err) - } - - // Check that we have files - files, ok := result["files"].([]interface{}) - if !ok { - t.Fatal("response should have files array") - } - - if len(files) == 0 { - t.Error("should have detected file changes") - } - - // Check counts exist - if _, ok := result["files_changed"]; !ok { - t.Error("response should have files_changed") - } - if _, ok := result["files_added"]; !ok { - t.Error("response should have files_added") - } -} - -// seedCompareVersions caches two versions of pkg:npm/test-compare with -// different archives for the compare endpoint. -func seedCompareVersions(t *testing.T, ts *testServer) { - t.Helper() - // Create two test archives with different content archive1Data := createArchiveWithContent(t, map[string]string{ "README.md": "# Version 1\n", @@ -584,52 +499,38 @@ func seedCompareVersions(t *testing.T, ts *testServer) { if err := ts.db.UpsertArtifact(artifact2); err != nil { t.Fatalf("failed to upsert artifact: %v", err) } -} -func compareTestVersions(ts *testServer) *httptest.ResponseRecorder { + // Test the compare endpoint req := httptest.NewRequest("GET", "/ui/api/compare/npm/test-compare/1.0.0/2.0.0", nil) w := httptest.NewRecorder() ts.handler.ServeHTTP(w, req) - return w -} -// Each comparison buffers both archives in memory, so only a few may run at -// once; the rest are refused instead of piling up until the process runs -// out of memory. -func TestCompareDiffRefusedWhileComparisonsAreRunning(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - seedCompareVersions(t, ts) - - for range maxConcurrentCompares { - if !ts.server.compares.tryAcquire() { - t.Fatal("could not take a free comparison slot") - } + if w.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) } - w := compareTestVersions(ts) - if w.Code != http.StatusServiceUnavailable { - t.Fatalf("status with all slots taken = %d, want %d", w.Code, http.StatusServiceUnavailable) - } - if w.Header().Get("Retry-After") == "" { - t.Error("refused comparison has no Retry-After header") + // Parse response + var result map[string]interface{} + if err := json.NewDecoder(w.Body).Decode(&result); err != nil { + t.Fatalf("failed to decode response: %v", err) } - ts.server.compares.release() - if w := compareTestVersions(ts); w.Code != http.StatusOK { - t.Fatalf("status after a slot was freed = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + // Check that we have files + files, ok := result["files"].([]interface{}) + if !ok { + t.Fatal("response should have files array") } -} -func TestCompareDiffReleasesItsSlot(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - seedCompareVersions(t, ts) + if len(files) == 0 { + t.Error("should have detected file changes") + } - for i := range maxConcurrentCompares + 1 { - if w := compareTestVersions(ts); w.Code != http.StatusOK { - t.Fatalf("comparison %d: status = %d, want %d: %s", i, w.Code, http.StatusOK, w.Body.String()) - } + // Check counts exist + if _, ok := result["files_changed"]; !ok { + t.Error("response should have files_changed") + } + if _, ok := result["files_added"]; !ok { + t.Error("response should have files_added") } } @@ -681,13 +582,12 @@ func TestHandleComparePage(t *testing.T) { body := w.Body.String() - // Check that versions are set correctly in JavaScript. The compare API takes - // each version as a path segment, so the page holds their escaped forms. - if !strings.Contains(body, "const fromVersionPath = '1.0.0'") { - t.Error("page should set fromVersionPath") + // Check that versions are set correctly in JavaScript + if !strings.Contains(body, "const fromVersion = '1.0.0'") { + t.Error("page should set fromVersion") } - if !strings.Contains(body, "const toVersionPath = '2.0.0'") { - t.Error("page should set toVersionPath") + if !strings.Contains(body, "const toVersion = '2.0.0'") { + t.Error("page should set toVersion") } // Test invalid format (missing separator) @@ -709,19 +609,25 @@ func TestHandleComparePage(t *testing.T) { } } -func TestOpenArchiveDetectsExtensionlessTarGz(t *testing.T) { - reader, err := openArchive("artifact", bytes.NewReader(createTestArchive(t)), "npm") - if err != nil { - t.Fatalf("openArchive failed: %v", err) +func TestArchiveFilename(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"package.tar.gz", "package.tar.gz"}, + {"d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856.zip"}, + {"file.zip", "file.zip"}, + {"archive.tgz", "archive.tgz"}, + {"noext", "noext.zip"}, } - defer func() { _ = reader.Close() }() - files, err := reader.List() - if err != nil { - t.Fatalf("List failed: %v", err) - } - if len(files) == 0 { - t.Fatal("expected files in extensionless archive") + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got := archiveFilename(tt.input) + if got != tt.want { + t.Errorf("archiveFilename(%q) = %q, want %q", tt.input, got, tt.want) + } + }) } } @@ -894,70 +800,3 @@ func createTarGzArchive(t *testing.T, files map[string]string) []byte { } return buf.Bytes() } - -// TestFirstBrowsableArtifact guards artifact selection against PEP 658 -// core-metadata sidecars. A sidecar resolves to the same version as the -// distribution it describes, so it is cached under that version, but it is plain -// text and openArchive cannot parse it. -func TestFirstBrowsableArtifact(t *testing.T) { - cached := func(filename string) database.Artifact { - return database.Artifact{ - Filename: filename, - StoragePath: sql.NullString{String: "pypi/" + filename, Valid: true}, - } - } - uncached := func(filename string) database.Artifact { - return database.Artifact{Filename: filename} - } - - tests := []struct { - name string - artifacts []database.Artifact - want string - }{ - {"no artifacts", nil, ""}, - { - "sidecar only is not browsable", - []database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata")}, - "", - }, - { - // '-' (0x2D) sorts before '.' (0x2E), so the sidecar precedes the - // sdist in the filename-ordered list the query returns. - "sidecar sorting ahead of the sdist is skipped", - []database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata"), cached("foo-1.0.tar.gz")}, - "foo-1.0.tar.gz", - }, - { - "sidecar skipped in favour of its own wheel", - []database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata"), cached("foo-1.0-py3-none-any.whl")}, - "foo-1.0-py3-none-any.whl", - }, - { - "uncached archive is still not selected", - []database.Artifact{cached("foo-1.0.tar.gz.metadata"), uncached("foo-1.0.tar.gz")}, - "", - }, - {"plain sdist", []database.Artifact{cached("foo-1.0.tar.gz")}, "foo-1.0.tar.gz"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := firstBrowsableArtifact(tt.artifacts) - - if tt.want == "" { - if got != nil { - t.Fatalf("firstBrowsableArtifact() = %q, want nil", got.Filename) - } - return - } - - if got == nil { - t.Fatalf("firstBrowsableArtifact() = nil, want %q", tt.want) - } - if got.Filename != tt.want { - t.Errorf("firstBrowsableArtifact() = %q, want %q", got.Filename, tt.want) - } - }) - } -} diff --git a/internal/server/cache_api.go b/internal/server/cache_api.go deleted file mode 100644 index 771d333..0000000 --- a/internal/server/cache_api.go +++ /dev/null @@ -1,86 +0,0 @@ -package server - -import ( - "net/http" - "strings" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/go-chi/chi/v5" -) - -// CacheDeleteResponse lists the cached files removed for a version. -type CacheDeleteResponse struct { - PURL string `json:"purl"` - Deleted []string `json:"deleted"` -} - -// handleCacheDelete handles DELETE /api/cache/{ecosystem}/{name}/{version} -// @Summary Delete cached artifacts -// @Description Removes the cached files of name@version, or only the one named by filename, so the next request fetches them from the upstream again. Pushed archives are not touched; delete those through /api/hosted. -// @Tags cache -// @Produce json -// @Param ecosystem path string true "Ecosystem" -// @Param name path string true "Package name as shown in the UI, e.g. com.google.guava:guava for Maven" -// @Param version path string true "Version" -// @Param filename query string false "Delete only this file" -// @Success 200 {object} CacheDeleteResponse -// @Failure 400 {object} ErrorResponse -// @Failure 404 {object} ErrorResponse -// @Failure 409 {object} ErrorResponse -// @Router /api/cache/{ecosystem}/{name}/{version} [delete] -func (s *Server) handleCacheDelete(w http.ResponseWriter, r *http.Request) { - segments, err := packagePathSegments(r) - if err != nil { - badRequest(w, err.Error()) - return - } - if len(segments) < minVersionPathSegments { - badRequest(w, "path must be /api/cache/{ecosystem}/{name}/{version}") - return - } - last := len(segments) - 1 - ecosystem, name, version := chi.URLParam(r, "ecosystem"), strings.Join(segments[:last], "/"), segments[last] - filename := r.URL.Query().Get("filename") - - versionPURL := s.cachedVersionPURL(ecosystem, name, version) - if versionPURL == "" { - notFound(w, "package not found") - return - } - artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL) - if err != nil { - s.logger.Error("failed to list artifacts", "purl", versionPURL, "error", err) - internalError(w, "failed to list artifacts") - return - } - - deleted := []string{} - for _, art := range artifacts { - if filename != "" && art.Filename != filename { - continue - } - if !art.StoragePath.Valid { - continue - } - if art.Source == database.SourceLocal { - if filename != "" { - writeError(w, http.StatusConflict, ErrCodeConflict, "pushed archives are deleted through /api/hosted") - return - } - continue - } - if err := s.db.DiscardArtifact(versionPURL, art.Filename, art.StoragePath.String); err != nil { - s.logger.Error("failed to delete cached artifact", "purl", versionPURL, "filename", art.Filename, "error", err) - internalError(w, "failed to delete cached artifact") - return - } - deleted = append(deleted, art.Filename) - } - if len(deleted) == 0 { - notFound(w, "no cached artifact matched") - return - } - - s.logger.Info("deleted cached artifacts", "purl", versionPURL, "files", deleted) - writeJSON(w, CacheDeleteResponse{PURL: versionPURL, Deleted: deleted}) -} diff --git a/internal/server/cache_api_test.go b/internal/server/cache_api_test.go deleted file mode 100644 index 8e66755..0000000 --- a/internal/server/cache_api_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package server - -import ( - "database/sql" - "encoding/json" - "net/http" - "net/http/httptest" - "slices" - "strings" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/database" - "github.com/go-chi/chi/v5" -) - -const ( - cacheTestPackage = "pkg:maven/com.example/lib" - cacheTestVersion = cacheTestPackage + "@1.0" -) - -// seedCacheDeleteVersion caches lib-1.0.jar and its .sha1 for -// com.example:lib 1.0, plus a pushed lib-1.0-local.jar when pushed is set. -func seedCacheDeleteVersion(t *testing.T, db *database.DB, pushed bool) { - t.Helper() - if err := db.UpsertPackage(&database.Package{PURL: cacheTestPackage, Ecosystem: "maven", Name: "com.example:lib"}); err != nil { - t.Fatalf("UpsertPackage: %v", err) - } - if err := db.UpsertVersion(&database.Version{PURL: cacheTestVersion, PackagePURL: cacheTestPackage}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - files := map[string]string{"lib-1.0.jar": database.SourceUpstream, "lib-1.0.jar.sha1": database.SourceUpstream} - if pushed { - files["lib-1.0-local.jar"] = database.SourceLocal - } - for filename, source := range files { - if err := db.UpsertArtifact(&database.Artifact{ - VersionPURL: cacheTestVersion, - Filename: filename, - UpstreamURL: "https://repo1.maven.org/maven2/com/example/lib/1.0/" + filename, - StoragePath: sql.NullString{String: "objects/" + filename, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, - Source: source, - }); err != nil { - t.Fatalf("UpsertArtifact %s: %v", filename, err) - } - } -} - -func cacheDelete(ts *testServer, target string) *httptest.ResponseRecorder { - r := chi.NewRouter() - r.Delete("/api/cache/{ecosystem}/*", ts.server.handleCacheDelete) - w := httptest.NewRecorder() - r.ServeHTTP(w, httptest.NewRequest(http.MethodDelete, target, nil)) - return w -} - -func cachedFiles(t *testing.T, db *database.DB) []string { - t.Helper() - artifacts, err := db.GetArtifactsByVersionPURL(cacheTestVersion) - if err != nil { - t.Fatalf("GetArtifactsByVersionPURL: %v", err) - } - var cached []string - for _, art := range artifacts { - if art.StoragePath.Valid { - cached = append(cached, art.Filename) - } - } - slices.Sort(cached) - return cached -} - -func queuedDeletes(t *testing.T, db *database.DB) []string { - t.Helper() - paths, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 100) - if err != nil { - t.Fatalf("GetDuePendingDeletes: %v", err) - } - slices.Sort(paths) - return paths -} - -func TestCacheDelete_OneFile(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - seedCacheDeleteVersion(t, ts.db, false) - - w := cacheDelete(ts, "/api/cache/maven/com.example:lib/1.0?filename=lib-1.0.jar") - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) - } - var resp CacheDeleteResponse - if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { - t.Fatalf("decoding response: %v", err) - } - if resp.PURL != cacheTestVersion || !slices.Equal(resp.Deleted, []string{"lib-1.0.jar"}) { - t.Errorf("response = %+v, want lib-1.0.jar deleted from %s", resp, cacheTestVersion) - } - if got := cachedFiles(t, ts.db); !slices.Equal(got, []string{"lib-1.0.jar.sha1"}) { - t.Errorf("still cached = %v, want only the .sha1", got) - } - if got := queuedDeletes(t, ts.db); !slices.Equal(got, []string{"objects/lib-1.0.jar"}) { - t.Errorf("queued for deletion = %v, want the jar's object", got) - } -} - -func TestCacheDelete_WholeVersionKeepsPushedFiles(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - seedCacheDeleteVersion(t, ts.db, true) - - w := cacheDelete(ts, "/api/cache/maven/com.example:lib/1.0") - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) - } - if got := cachedFiles(t, ts.db); !slices.Equal(got, []string{"lib-1.0-local.jar"}) { - t.Errorf("still cached = %v, want only the pushed jar", got) - } - if got := queuedDeletes(t, ts.db); !slices.Equal(got, []string{"objects/lib-1.0.jar", "objects/lib-1.0.jar.sha1"}) { - t.Errorf("queued for deletion = %v, want both upstream objects", got) - } -} - -func TestCacheDelete_Refusals(t *testing.T) { - tests := []struct { - name string - target string - want int - }{ - {"pushed file", "/api/cache/maven/com.example:lib/1.0?filename=lib-1.0-local.jar", http.StatusConflict}, - {"unknown file", "/api/cache/maven/com.example:lib/1.0?filename=other.jar", http.StatusNotFound}, - {"unknown version", "/api/cache/maven/com.example:lib/2.0", http.StatusNotFound}, - {"unknown package", "/api/cache/maven/com.example:other/1.0", http.StatusNotFound}, - {"no version", "/api/cache/maven/lib", http.StatusBadRequest}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - seedCacheDeleteVersion(t, ts.db, true) - - w := cacheDelete(ts, tt.target) - if w.Code != tt.want { - t.Fatalf("status = %d, want %d; body: %s", w.Code, tt.want, w.Body.String()) - } - if got := cachedFiles(t, ts.db); len(got) != 3 { - t.Errorf("still cached = %v, want all three files untouched", got) - } - }) - } -} - -func TestVersionPage_CacheDeleteButtons(t *testing.T) { - for _, enabled := range []bool{false, true} { - ts := newTestServer(t) - seedCacheDeleteVersion(t, ts.db, true) - ts.server.cfg.CacheDelete = enabled - - w := httptest.NewRecorder() - ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/package/maven/com.example:lib/1.0", nil)) - ts.close() - if w.Code != http.StatusOK { - t.Fatalf("enabled=%v: status = %d, want 200", enabled, w.Code) - } - body := w.Body.String() - // One button per upstream file plus the version-wide one; the pushed - // jar gets none. - wantButtons := 0 - if enabled { - wantButtons = 3 - } - if got := strings.Count(body, `class="cache-delete `); got != wantButtons { - t.Errorf("enabled=%v: %d delete buttons, want %d", enabled, got, wantButtons) - } - if enabled && !strings.Contains(body, `data-cache-url="/ui/api/cache/maven/com.example:lib/1.0"`) { - t.Errorf("page has no data-cache-url for the version") - } - if got := strings.Contains(body, ` -{{end}} diff --git a/internal/server/templates/pages/browse_source.html b/internal/server/templates/pages/browse_source.html index e43324b..ca06652 100644 --- a/internal/server/templates/pages/browse_source.html +++ b/internal/server/templates/pages/browse_source.html @@ -7,7 +7,7 @@ / {{.PackageName}} / - {{.Version}} + {{.Version}} / Browse Source @@ -51,10 +51,7 @@ diff --git a/internal/server/templates/pages/compare_versions.html b/internal/server/templates/pages/compare_versions.html index 79cd89b..32803421 100644 --- a/internal/server/templates/pages/compare_versions.html +++ b/internal/server/templates/pages/compare_versions.html @@ -66,22 +66,20 @@ - + - + + {{end}} diff --git a/internal/server/templates/pages/project_show.html b/internal/server/templates/pages/project_show.html deleted file mode 100644 index 2d1b56a..0000000 --- a/internal/server/templates/pages/project_show.html +++ /dev/null @@ -1,59 +0,0 @@ -{{define "title"}}{{.Project.Name}} - git-pkgs proxy{{end}} - -{{define "content"}} -
- ← Projects -

{{.Project.Name}}

-

- {{.Project.Components}} packages, uploaded {{.Project.UploadedAt}}{{if .Project.Branch}} from {{.Project.Branch}}{{end}}{{if .Project.Ref}} at {{.Project.Ref}}{{end}} -

- {{if .Outdated}} -
- {{.Outdated}} outdated -
- {{end}} -
- - - -
-
- - -{{if .Components}} -
-
- {{range .Components}} -
-
- {{template "ecosystem_badge" .Ecosystem}} - {{if .Cached}} - {{.Name}} - {{else}} - {{.Name}} - {{end}} - {{.Version}} -
-
- {{if .Outdated}} - outdated, latest {{.LatestVersion}} - {{end}} - {{if not .Cached}} - not cached - {{end}} -
-
- {{end}} -
-
-{{else}} -
-

The latest SBOM lists no packages the proxy serves

-
-{{end}} -{{end}} diff --git a/internal/server/templates/pages/projects_list.html b/internal/server/templates/pages/projects_list.html deleted file mode 100644 index d7256da..0000000 --- a/internal/server/templates/pages/projects_list.html +++ /dev/null @@ -1,33 +0,0 @@ -{{define "title"}}Projects - git-pkgs proxy{{end}} - -{{define "content"}} -
-

Projects

-

{{len .Rows}} projects, as of their latest SBOM upload

-
- -{{if .Rows}} -
-
- {{range .Rows}} -
-
- {{.Name}} - {{if or .Branch .Ref}} -
{{.Branch}}{{if and .Branch .Ref}} @ {{end}}{{.Ref}}
- {{end}} -
-
- {{.Components}} packages - {{.UploadedAt}} -
-
- {{end}} -
-
-{{else}} -
-

No project has uploaded an SBOM yet

-
-{{end}} -{{end}} diff --git a/internal/server/templates/pages/version_show.html b/internal/server/templates/pages/version_show.html index 468f24c..a30d8b5 100644 --- a/internal/server/templates/pages/version_show.html +++ b/internal/server/templates/pages/version_show.html @@ -1,4 +1,4 @@ -{{define "title"}}{{.Version.DisplayPURL}} - git-pkgs proxy{{end}} +{{define "title"}}{{.Package.Name}}@{{.Version.PURL}} - git-pkgs proxy{{end}} {{define "content"}}
@@ -9,7 +9,7 @@
{{template "ecosystem_badge" .Package.Ecosystem}} -

{{.Version.DisplayPURL}}

+

{{.Version.PURL}}

{{if .IsOutdated}} outdated {{end}} @@ -22,7 +22,7 @@ {{end}} {{if .HasCachedArtifact}} {{if .Artifacts}} -
-
+
+

Artifacts ({{len .Artifacts}})

- {{if and .CacheDelete .HasDeletableArtifact}} - - {{end}}
-
{{range .Artifacts}}
@@ -85,9 +81,6 @@
{{if .Size.Valid}}{{.Size.Int64}} bytes{{end}} {{if gt .HitCount 0}}{{.HitCount}} cache hits{{end}} - {{if and $.CacheDelete .StoragePath.Valid (ne .Source "local")}} - - {{end}}
{{if .ContentHash.Valid}} @@ -107,9 +100,6 @@ {{end}}
-{{if .CacheDelete}} - -{{end}} {{else}}
No artifacts cached yet diff --git a/internal/server/templates_handlers_test.go b/internal/server/templates_handlers_test.go deleted file mode 100644 index 1e3f1f2..0000000 --- a/internal/server/templates_handlers_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package server - -import ( - "io/fs" - "regexp" - "testing" -) - -// inlineHandlerWithData matches an inline event handler attribute whose value -// interpolates a JavaScript template expression. The browser decodes HTML -// entities in the attribute before running the handler, so escapeHTML cannot -// keep a quote in the interpolated value from ending the handler's string. -var inlineHandlerWithData = regexp.MustCompile(`\son[a-z]+\s*=\s*"[^"]*\$\{`) - -func TestTemplatesInterpolateNoDataIntoInlineHandlers(t *testing.T) { - err := fs.WalkDir(templatesFS, "templates", func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return err - } - source, err := fs.ReadFile(templatesFS, path) - if err != nil { - return err - } - for _, match := range inlineHandlerWithData.FindAll(source, -1) { - t.Errorf("%s: inline handler interpolates data: %s", path, match) - } - return nil - }) - if err != nil { - t.Fatal(err) - } -} diff --git a/internal/server/templates_test.go b/internal/server/templates_test.go index c6fa492..e9a4967 100644 --- a/internal/server/templates_test.go +++ b/internal/server/templates_test.go @@ -2,13 +2,10 @@ package server import ( "database/sql" - "io/fs" "net/http" "net/http/httptest" - "regexp" "strings" "testing" - "time" "github.com/git-pkgs/proxy/internal/database" ) @@ -40,45 +37,6 @@ func TestTemplatesRenderAllPages(t *testing.T) { {Ecosystem: "cargo", Name: "serde", Version: "1.0.0", Size: "200 KB", CachedAt: "1 hour ago"}, }, }}, - {"analytics", AnalyticsData{ - Totals: AnalyticsTotals{ - DownloadedBytes: 1_500_000_000, - Downloaded: "1.4 GB", - Downloads: "12,004", - CacheSize: "420.0 MB", - CachedArtifacts: "1,204", - Packages: "310", - Versions: "902", - Ecosystems: 3, - ActiveEcosystems: 2, - Amplification: "3.4x", - }, - EnrichmentStats: EnrichmentStatsView{TotalVulnerabilities: 3, CriticalVulns: 1, HasVulns: true}, - Ecosystems: []EcosystemRow{ - {Ecosystem: "npm", DownloadedBytes: 1_000_000_000, Downloaded: "953.7 MB", Downloads: "9,000", CacheSize: "300.0 MB", AvgArtifactSize: "120 KB", Artifacts: "900", Packages: "200", Versions: "700", SharePct: "66.7"}, - {Ecosystem: "cargo", DownloadedBytes: 500_000_000, Downloaded: "476.8 MB", Downloads: "3,004", CacheSize: "120.0 MB", AvgArtifactSize: "400 KB", Artifacts: "300", Packages: "100", Versions: "200", SharePct: "33.3"}, - {Ecosystem: "rpm", Downloaded: "0 B", Downloads: "0", CacheSize: "0 B", AvgArtifactSize: "0 B", Artifacts: "0", Packages: "10", Versions: "2", SharePct: "0.0"}, - }, - Donut: donutView([]database.EcosystemStats{ - {Ecosystem: "npm", CacheSize: 300, Downloads: 9000, DownloadedBytes: 1_000_000_000}, - {Ecosystem: "cargo", CacheSize: 120, Downloads: 3004, DownloadedBytes: 500_000_000}, - }, 1_500_000_000, "1.4 GB"), - Runtime: RuntimeView{ - Available: true, - Requests: "12,004", - ActiveRequests: "2", - RequestMean: "8.1 ms", - ArtifactCache: CacheLookups{Hits: "9,000", Misses: "1,000", HitRatio: "90.0", HasTraffic: true}, - MetadataCache: CacheLookups{Hits: "4,000", Misses: "500", HitRatio: "88.9", HasTraffic: true}, - MetadataServed: "12.0 MB", - StatusClasses: []LabelledCount{{Label: "2xx", Count: "11,900"}, {Label: "5xx", Count: "104", Bad: true}}, - StorageOps: []LabelledStat{{Label: "get", Count: "9,000", Mean: "412 µs"}}, - ScanErrors: []LabelledCount{{Label: "npm · clamav · timeout", Count: "3", Bad: true}}, - ScanningOn: true, - Breakers: []BreakerRow{{Registry: "registry.npmjs.org", State: "closed"}}, - }, - }}, - {"analytics", AnalyticsData{}}, {"install", struct { Layout BaseURL string @@ -105,15 +63,13 @@ func TestTemplatesRenderAllPages(t *testing.T) { TotalPages: 0, }}, {"packages_list", PackagesListPageData{ - Ecosystem: "", - SortBy: defaultSortBy, - Results: []SearchResultItem{{Ecosystem: "npm", Name: "express", Hits: 200, SizeFormatted: "2 MB"}}, - Count: 1, - TotalPackages: 1, - EcosystemFilters: []EcosystemFilter{{Ecosystem: "npm", Count: 1}}, - Page: 1, - PerPage: 50, - TotalPages: 1, + Ecosystem: "", + SortBy: defaultSortBy, + Results: []SearchResultItem{{Ecosystem: "npm", Name: "express", Hits: 200, SizeFormatted: "2 MB"}}, + Count: 1, + Page: 1, + PerPage: 50, + TotalPages: 1, }}, {"package_show", PackageShowData{ Package: &database.Package{ @@ -123,12 +79,8 @@ func TestTemplatesRenderAllPages(t *testing.T) { LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, License: sql.NullString{String: "MIT", Valid: true}, }, - Versions: []PackageVersion{ - {Version: database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}}, - { - Version: database.Version{PURL: "pkg:npm/lodash@4.17.22", PackagePURL: "pkg:npm/lodash"}, - AvailableAt: time.Date(2026, 1, 2, 3, 4, 0, 0, time.UTC), - }, + Versions: []database.Version{ + {PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}, }, Vulnerabilities: []database.Vulnerability{}, LicenseCategory: "permissive", @@ -139,7 +91,7 @@ func TestTemplatesRenderAllPages(t *testing.T) { Ecosystem: "npm", Name: "minimal", }, - Versions: []PackageVersion{}, + Versions: []database.Version{}, Vulnerabilities: []database.Vulnerability{}, LicenseCategory: "unknown", }}, @@ -234,64 +186,6 @@ func TestRenderEmitsCanonicalAndOG(t *testing.T) { } } -func TestFooterUsesBuildInfoWhenPageDefinesVersion(t *testing.T) { - templates := &Templates{} - buildInfo := BuildInfo{Version: "proxy-build-1.2.3", Commit: "abc123def"} - wantFooter := "proxy proxy-build-1.2.3 (abc123def)" - - tests := []struct { - name string - page string - data any - shadow string - }{ - { - name: "version show page", - page: "version_show", - data: VersionShowData{ - Layout: Layout{BuildInfo: buildInfo}, - Package: &database.Package{ - PURL: "pkg:npm/lodash", - Ecosystem: "npm", - Name: "lodash", - }, - Version: &database.Version{ - PURL: "pkg:npm/lodash@9.9.9", - PackagePURL: "pkg:npm/lodash", - }, - }, - shadow: "9.9.9", - }, - { - name: "browse source page", - page: "browse_source", - data: BrowseSourceData{ - Layout: Layout{BuildInfo: buildInfo}, - Ecosystem: "npm", - PackageName: "lodash", - Version: "9.9.9", - }, - shadow: "9.9.9", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - w := httptest.NewRecorder() - if err := templates.Render(w, tt.page, tt.data); err != nil { - t.Fatalf("Render(%q) failed: %v", tt.page, err) - } - body := w.Body.String() - if !strings.Contains(body, wantFooter) { - t.Errorf("footer missing build info %q", wantFooter) - } - if strings.Contains(body, "proxy "+tt.shadow) { - t.Errorf("footer used page Version %q instead of BuildInfo", tt.shadow) - } - }) - } -} - func TestRenderOmitsCanonicalWhenUIBaseURLUnset(t *testing.T) { templates := &Templates{} @@ -567,35 +461,6 @@ func TestEcosystemBadgeLabel(t *testing.T) { } } -func TestOCIRegistryInstructionsDockerPull(t *testing.T) { - registries := getRegistryConfigs("http://package-proxy:8080") - for _, registry := range registries { - if registry.ID != "oci" { - continue - } - want := "docker pull package-proxy:8080/library/nginx:latest" - if !strings.Contains(string(registry.Instructions), want) { - t.Errorf("OCI instructions = %q, want substring %q", registry.Instructions, want) - } - return - } - t.Fatal("OCI registry instructions not found") -} - -func TestSwiftRegistryInstructionsAllowLocalHTTP(t *testing.T) { - registries := getRegistryConfigs("http://localhost:8080") - for _, registry := range registries { - if registry.ID != "swift" { - continue - } - if !strings.Contains(string(registry.Instructions), "--allow-insecure-http") { - t.Error("Swift HTTP instructions do not allow the insecure local registry") - } - return - } - t.Fatal("Swift registry instructions not found") -} - func TestEcosystemBadgeClasses(t *testing.T) { // Every supported ecosystem should return a non-empty class string ecosystems := supportedEcosystems() @@ -616,39 +481,6 @@ func TestEcosystemBadgeClasses(t *testing.T) { } } -func TestBuildEcosystemFilters(t *testing.T) { - filters := buildEcosystemFilters(map[string]int64{ - "npm": 3, - "cargo": 2, - "empty": 0, - }) - - if len(filters) != 2 { - t.Fatalf("expected 2 filters, got %d", len(filters)) - } - if filters[0].Ecosystem != "cargo" || filters[0].Count != 2 { - t.Errorf("first filter = %#v, want cargo with count 2", filters[0]) - } - if filters[1].Ecosystem != "npm" || filters[1].Count != 3 { - t.Errorf("second filter = %#v, want npm with count 3", filters[1]) - } - - extra := buildEcosystemFilters(map[string]int64{"custom": 1}) - if len(extra) != 1 || extra[0].Ecosystem != "custom" { - t.Errorf("unexpected extra filters: %#v", extra) - } -} - -func TestEcosystemPillClasses(t *testing.T) { - classes := ecosystemPillClasses("npm") - if !strings.Contains(classes, "rounded-full") { - t.Error("pill classes should use rounded-full") - } - if !strings.Contains(classes, "bg-red-100") { - t.Error("npm pill should use npm colors") - } -} - func TestFormatSize(t *testing.T) { tests := []struct { bytes int64 @@ -715,28 +547,3 @@ func BenchmarkFirstRender(b *testing.B) { } } } - -// thirdPartyAsset matches a script or stylesheet loaded from another origin. -var thirdPartyAsset = regexp.MustCompile(`<(?:script[^>]*\ssrc|link[^>]*\shref)="(?:https?:)?//[^"]*"`) - -// Pages run on the proxy origin, so a script or stylesheet from elsewhere -// would run with the proxy's privileges and break offline deployments. -// Vendor it under static/vendor instead. -func TestTemplatesLoadNoThirdPartyAssets(t *testing.T) { - err := fs.WalkDir(templatesFS, "templates", func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return err - } - content, err := templatesFS.ReadFile(path) - if err != nil { - return err - } - for _, match := range thirdPartyAsset.FindAllString(string(content), -1) { - t.Errorf("%s loads a third-party asset: %s", path, match) - } - return nil - }) - if err != nil { - t.Fatal(err) - } -} diff --git a/internal/server/transport_test.go b/internal/server/transport_test.go deleted file mode 100644 index 7464282..0000000 --- a/internal/server/transport_test.go +++ /dev/null @@ -1,190 +0,0 @@ -package server - -import ( - "crypto/tls" - "io" - "net" - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/registries/safehttp" -) - -// tlsUpstream starts a TLS test server that counts accepted connections. -func tlsUpstream(t *testing.T, handler http.HandlerFunc) (*httptest.Server, func() int) { - t.Helper() - var mu sync.Mutex - accepted := 0 - srv := httptest.NewUnstartedServer(handler) - srv.Config.ConnState = func(_ net.Conn, state http.ConnState) { - if state == http.StateNew { - mu.Lock() - accepted++ - mu.Unlock() - } - } - srv.StartTLS() - t.Cleanup(srv.Close) - return srv, func() int { - mu.Lock() - defer mu.Unlock() - return accepted - } -} - -// trustUpstream makes transport trust srv's certificate and pins HTTP/1.1 so -// every in-flight request needs its own connection. -func trustUpstream(t *testing.T, transport *http.Transport, srv *httptest.Server) { - t.Helper() - transport.TLSClientConfig = &tls.Config{ - RootCAs: srv.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs, - NextProtos: []string{"http/1.1"}, - MinVersion: tls.VersionTLS12, - } - transport.ForceAttemptHTTP2 = false - t.Cleanup(transport.CloseIdleConnections) -} - -// burst issues n concurrent GETs and drains every body. The transport hands a -// connection back to the idle pool before the body's final Read returns, so -// the pool is settled when burst returns. -func burst(t *testing.T, client *http.Client, url string, n int) { - t.Helper() - var wg sync.WaitGroup - errs := make(chan error, n) - for range n { - wg.Add(1) - go func() { - defer wg.Done() - resp, err := client.Get(url) - if err != nil { - errs <- err - return - } - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - }() - } - wg.Wait() - close(errs) - for err := range errs { - t.Errorf("burst request: %v", err) - } -} - -// TestUpstreamClientReusesConnectionsAcrossBursts measures how many -// connections a second burst of concurrent requests reuses. With Go's default -// of two idle connections per host most of them are re-dialled; with the -// tuned transport the second burst reuses all of them. -func TestUpstreamClientReusesConnectionsAcrossBursts(t *testing.T) { - const burstSize = 8 - - // holdBurst returns a handler that answers a request only once burstSize - // of them are waiting at the same time. With HTTP/1.1 pinned that puts - // every burst on burstSize distinct connections, whatever the scheduling. - holdBurst := func() http.HandlerFunc { - var mu sync.Mutex - waiting := 0 - release := make(chan struct{}) - return func(w http.ResponseWriter, r *http.Request) { - mu.Lock() - gate := release - waiting++ - if waiting == burstSize { - close(gate) - waiting = 0 - release = make(chan struct{}) - } - mu.Unlock() - select { - case <-gate: - case <-r.Context().Done(): - } - _, _ = w.Write([]byte("ok")) - } - } - - tests := []struct { - name string - client *http.Client - // Bounds on how many connections the second burst has to dial. - minNew, maxNew int - }{ - { - name: "go default keeps two idle connections", - client: safehttp.New(nil, safehttp.Options{AllowLoopback: true}), - minNew: burstSize - 2, - maxNew: burstSize, - }, - { - name: "tuned transport reuses the whole burst", - client: newUpstreamClient(config.UpstreamConfig{AllowLoopback: true}), - minNew: 0, - maxNew: 0, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - srv, accepted := tlsUpstream(t, holdBurst()) - transport := tc.client.Transport.(*http.Transport) - trustUpstream(t, transport, srv) - - burst(t, tc.client, srv.URL, burstSize) - afterFirst := accepted() - if afterFirst < burstSize { - t.Fatalf("first burst opened %d connections, want at least %d", afterFirst, burstSize) - } - - burst(t, tc.client, srv.URL, burstSize) - newInSecond := accepted() - afterFirst - t.Logf("second burst: %d new connections, %d reused", newInSecond, burstSize-newInSecond) - - if newInSecond < tc.minNew || newInSecond > tc.maxNew { - t.Errorf("second burst opened %d new connections, want between %d and %d", newInSecond, tc.minNew, tc.maxNew) - } - }) - } -} - -// TestUpstreamClientBoundsStallBeforeHeaders pins the production transport -// values, then lowers the header timeout so it can show within milliseconds -// that this is what cuts off an upstream which accepts a request but never -// sends headers. -func TestUpstreamClientBoundsStallBeforeHeaders(t *testing.T) { - client := newUpstreamClient(config.UpstreamConfig{AllowLoopback: true}) - transport := client.Transport.(*http.Transport) - if transport.MaxIdleConnsPerHost != upstreamMaxIdleConnsPerHost { - t.Fatalf("MaxIdleConnsPerHost = %d, want %d", transport.MaxIdleConnsPerHost, upstreamMaxIdleConnsPerHost) - } - if transport.ResponseHeaderTimeout != upstreamResponseHeaderTimeout { - t.Fatalf("ResponseHeaderTimeout = %v, want %v", transport.ResponseHeaderTimeout, upstreamResponseHeaderTimeout) - } - - stall := make(chan struct{}) - srv, _ := tlsUpstream(t, func(_ http.ResponseWriter, r *http.Request) { - select { - case <-stall: - case <-r.Context().Done(): - } - }) - t.Cleanup(func() { close(stall) }) - trustUpstream(t, transport, srv) - - // Far below the client's overall timeout, so the header timeout ends the - // request; the error text tells the two timeouts apart. - transport.ResponseHeaderTimeout = 200 * time.Millisecond - - resp, err := client.Get(srv.URL) - if err == nil { - _ = resp.Body.Close() - t.Fatal("request to a stalled upstream succeeded, want a timeout") - } - if !strings.Contains(err.Error(), "timeout awaiting response headers") { - t.Fatalf("error = %v, want a response-header timeout", err) - } -} diff --git a/internal/server/untrusted_content_test.go b/internal/server/untrusted_content_test.go deleted file mode 100644 index 5982402..0000000 --- a/internal/server/untrusted_content_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package server - -import ( - "context" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "testing" - - "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/handler" - "github.com/git-pkgs/proxy/internal/storage" - "github.com/git-pkgs/registries/fetch" - "github.com/go-chi/chi/v5" -) - -// TestPackageRoutesSandboxUpstreamContent checks that responses on package -// routes cannot run as pages on the proxy's origin, which also serves /ui. -// Their bodies and Content-Type come from upstreams, or from whoever pushed a -// hosted package, so an archive served as text/html would otherwise run its -// scripts with access to the UI. -func TestPackageRoutesSandboxUpstreamContent(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "text/html") - _, _ = io.WriteString(w, "") - })) - t.Cleanup(upstream.Close) - - db, err := database.Create(t.TempDir() + "/test.db") - if err != nil { - t.Fatalf("creating database: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) - if err != nil { - t.Fatalf("opening storage: %v", err) - } - fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) - t.Cleanup(func() { _ = fetcher.Close() }) - proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil))) - proxy.HTTPClient = upstream.Client() - - cfg := config.Default() - cfg.BaseURL = "http://proxy.example" - cfg.Upstream.Generic = map[string]string{"site": upstream.URL} - cfg.Upstream.APK = map[string]string{"alpine": upstream.URL} - s := &Server{cfg: cfg} - r := chi.NewRouter() - s.mountProtocolHandlers(r, proxy) - - for _, target := range []string{ - "/generic/site/files/page-1.0.html", // cached artifact - "/generic/site/files/page-1.0.html", // the same, from the cache - "/apk/alpine/v3.22/main/x86_64/README.html", // relayed as is - "/npm/does-not-exist%2F..", // an error response - } { - w := httptest.NewRecorder() - r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) - if got := w.Header().Get("X-Content-Type-Options"); got != "nosniff" { - t.Errorf("%s (status %d): X-Content-Type-Options = %q, want nosniff", target, w.Code, got) - } - if got := w.Header().Get("Content-Security-Policy"); got != "sandbox" { - t.Errorf("%s (status %d): Content-Security-Policy = %q, want sandbox", target, w.Code, got) - } - } -} diff --git a/internal/server/upstream_auth_redirect_test.go b/internal/server/upstream_auth_redirect_test.go deleted file mode 100644 index 7c2f0c4..0000000 --- a/internal/server/upstream_auth_redirect_test.go +++ /dev/null @@ -1,125 +0,0 @@ -package server - -import ( - "context" - "errors" - "io" - "log/slog" - "net" - "net/http" - "net/http/httptest" - "path/filepath" - "sync" - "testing" - "time" - - "github.com/git-pkgs/proxy/internal/config" -) - -const redirectAuthHeader = "X-Api-Key" - -// headerRecorder remembers the auth header value each request carried. -type headerRecorder struct { - mu sync.Mutex - values []string -} - -func (h *headerRecorder) record(r *http.Request) { - h.mu.Lock() - defer h.mu.Unlock() - h.values = append(h.values, r.Header.Get(redirectAuthHeader)) -} - -func (h *headerRecorder) seen() []string { - h.mu.Lock() - defer h.mu.Unlock() - return append([]string(nil), h.values...) -} - -func TestHeaderAuthIsNotForwardedAcrossRedirects(t *testing.T) { - var atOther headerRecorder - other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - atOther.record(r) - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"name":"leftpad","versions":{}}`) - })) - defer other.Close() - - var atRegistry headerRecorder - registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - atRegistry.record(r) - http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound) - })) - defer registry.Close() - - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatalf("reserving proxy address: %v", err) - } - t.Cleanup(func() { _ = listener.Close() }) - - tempDir := t.TempDir() - cfg := config.Default() - cfg.Listen = listener.Addr().String() - cfg.BaseURL = "http://" + cfg.Listen - cfg.Database.Path = filepath.Join(tempDir, "proxy.db") - cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") - cfg.Upstream.AllowLoopback = true - cfg.Upstream.NPM = registry.URL - cfg.Upstream.Auth = map[string]config.AuthConfig{ - registry.URL: {Type: "header", HeaderName: redirectAuthHeader, HeaderValue: "registry-secret"}, - } - if err := cfg.Validate(); err != nil { - t.Fatalf("validating config: %v", err) - } - - proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{Version: "test", Commit: "test"}) - if err != nil { - t.Fatalf("creating server: %v", err) - } - startErr := make(chan error, 1) - go func() { startErr <- proxyServer.Start(listener) }() - defer func() { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := proxyServer.Shutdown(ctx); err != nil { - t.Errorf("shutting down server: %v", err) - } - if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { - t.Errorf("Start() error = %v, want %v", err, http.ErrServerClosed) - } - }() - - resp := getWhenListening(t, cfg.BaseURL+"/npm/leftpad") - _ = resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) - } - - registrySeen := atRegistry.seen() - if len(registrySeen) == 0 || registrySeen[0] != "registry-secret" { - t.Errorf("registry received %s = %q, want the configured value", redirectAuthHeader, registrySeen) - } - for _, value := range atOther.seen() { - if value != "" { - t.Errorf("redirect target received %s = %q, want none", redirectAuthHeader, value) - } - } -} - -// getWhenListening retries a GET until the proxy accepts connections. -func getWhenListening(t *testing.T, url string) *http.Response { - t.Helper() - client := &http.Client{Timeout: 5 * time.Second} - deadline := time.Now().Add(5 * time.Second) - for { - resp, err := client.Get(url) - if err == nil { - return resp - } - if time.Now().After(deadline) { - t.Fatalf("proxy did not start: %v", err) - } - time.Sleep(10 * time.Millisecond) - } -} diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 81af78f..67e91d0 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -22,29 +22,11 @@ import ( const osWindows = "windows" -// Upload buffering for cloud backends. The S3 driver otherwise allocates -// (5+1) x 8 MiB part buffers up front for every object over 16 MiB, and the -// Azure driver uses 5 x 8 MiB blocks; with several large artifacts being -// cached at once that adds up to hundreds of MiB. 5 MiB is the smallest part -// S3 accepts. The file backend ignores both. -const ( - uploadBufferSize = 5 << 20 - uploadConcurrency = 2 -) - -// attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an -// earlier version wrote. -const attrsExt = ".attrs" - // Blob implements Storage using gocloud.dev/blob. // Supports local filesystem (file://) and S3 (s3://) URLs. type Blob struct { bucket *blob.Bucket url string - - // fileRoot is the directory backing a file:// bucket, empty for cloud - // backends. Used only to clear sidecars an earlier version wrote. - fileRoot string } // OpenBucket opens a blob bucket from a URL. @@ -53,20 +35,9 @@ type Blob struct { // - file:///path/to/dir - Local filesystem storage // - s3://bucket-name - Amazon S3 (uses AWS_* environment variables) // - s3://bucket-name?region=us-east-1&endpoint=http://localhost:9000 - S3-compatible (MinIO, etc.) -// - gs://bucket-name - Google Cloud Storage (uses Application Default Credentials; -// supports Workload Identity on GKE/GCE without any extra configuration) -// - azblob://container-name - Azure Blob Storage // // For local filesystem, the directory is created if it doesn't exist. -// -//nolint:ireturn // The URL scheme selects the storage implementation. -func OpenBucket(ctx context.Context, urlStr string) (Storage, error) { - if strings.HasPrefix(urlStr, "gs://") { - return OpenGCS(ctx, urlStr) - } - - var fileRoot string - +func OpenBucket(ctx context.Context, urlStr string) (*Blob, error) { // Handle file:// URLs specially to create the directory if strings.HasPrefix(urlStr, "file://") { path := strings.TrimPrefix(urlStr, "file://") @@ -94,8 +65,6 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) { return nil, fmt.Errorf("resolving path: %w", err) } - fileRoot = absPath - // Convert back to URL format with forward slashes urlPath := filepath.ToSlash(absPath) if runtime.GOOS == osWindows { @@ -109,14 +78,7 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) { // This avoids "invalid cross-device link" errors from os.Rename when // the bucket directory and os.TempDir are on different filesystems // (e.g. Docker volume mounts). - // - // Do not write fileblob's ".attrs" sidecar. It is rewritten with - // os.Create, truncating in place outside the atomic rename that - // protects the blob, so a read overlapping a write can decode a - // partial file; a missing one defaults cleanly, a truncated one does - // not. Nothing in the proxy needs it: Store sets no ContentType, and - // Size reads os.Stat via Attributes. - urlStr += "?no_tmp_dir=true&metadata=skip" + urlStr += "?no_tmp_dir=true" } bucket, err := blob.OpenBucket(ctx, urlStr) @@ -124,116 +86,22 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) { return nil, fmt.Errorf("opening bucket: %w", err) } - return &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot}, nil -} - -// legacySidecarPath gives the ".attrs" path an earlier version wrote for key, -// or "" when that path would not be a file inside fileRoot. -func (b *Blob) legacySidecarPath(key string) string { - if p := b.localPath(key); p != "" { - return p + attrsExt - } - return "" -} - -// localPath gives the file fileblob keeps key in, or "" when that would not -// be a file inside fileRoot. -// -// The key is escaped the way fileblob escapes it on the way to disk. -// filepath.Localize then validates the escaped form: it rejects an empty, -// absolute or ".." path, and "." would name fileRoot itself. What it declines -// are keys the proxy never produces. -func (b *Blob) localPath(key string) string { - if b.fileRoot == "" { - return "" - } - rel, err := filepath.Localize(escapeKey(key)) - if err != nil || rel == "." { - return "" - } - return filepath.Join(b.fileRoot, rel) -} - -// escapeKey mirrors fileblob's unexported escapeKey, which hex-escapes a rune -// as "__0x__". Slashes stay as "/" for filepath.Localize to convert. -func escapeKey(key string) string { - runes := []rune(key) - var out strings.Builder - for i, r := range runes { - if escapeRune(runes, i) { - fmt.Fprintf(&out, "__%#x__", r) - } else { - out.WriteRune(r) - } - } - return out.String() -} - -// escapeRune is fileblob's rule for which runes of a key to escape: control -// characters, a raw path separator, a slash that would form "../", "//" or -// end the key, and on Windows the characters its filesystem reserves. -func escapeRune(r []rune, i int) bool { - c := r[i] - switch { - case c < ' ': - return true - case os.PathSeparator != '/' && c == os.PathSeparator: - return true - case i > 1 && c == '/' && r[i-1] == '.' && r[i-2] == '.': - return true - case i > 0 && c == '/' && r[i-1] == '/': - return true - case c == '/' && i == len(r)-1: - return true - case os.PathSeparator == '\\' && strings.ContainsRune(`<>:"|?*`, c): - return true - } - return false -} - -// clearLegacySidecar removes the ".attrs" file an earlier version wrote for -// key. Nothing rewrites one now, so a sidecar left partial by an interrupted -// write would fail every read of that key for good. Removing is atomic where -// the rewrite was not, so a concurrent reader gets the whole old file or -// nothing. -// -// Failure is deliberately not fatal. Usually the key never had a sidecar and -// os.Remove reports not-exist. A real failure leaves exactly the state this -// change inherited, while failing the write would turn a cleanup miss into a -// failed request. Windows makes that concrete: Go opens files without -// FILE_SHARE_DELETE, so a reader holding the sidecar open blocks deletion, and -// that reader is the very workload this change protects. The next store of the -// key retries. -func (b *Blob) clearLegacySidecar(key string) { - if sidecar := b.legacySidecarPath(key); sidecar != "" { - _ = os.Remove(sidecar) - } + return &Blob{bucket: bucket, url: urlStr}, nil } func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { - b.clearLegacySidecar(path) - // Compute hash while writing h := sha256.New() tee := io.TeeReader(r, h) - // gocloud commits whatever was written when the writer closes, unless - // its context is cancelled first, so a failed copy cancels it to abort. - writeCtx, abort := context.WithCancel(ctx) - defer abort() - - opts := &blob.WriterOptions{ - BufferSize: uploadBufferSize, - MaxConcurrency: uploadConcurrency, - } - w, err := b.bucket.NewWriter(writeCtx, path, opts) + opts := &blob.WriterOptions{} + w, err := b.bucket.NewWriter(ctx, path, opts) if err != nil { return 0, "", fmt.Errorf("creating writer: %w", err) } size, err := io.Copy(w, tee) if err != nil { - abort() _ = w.Close() return 0, "", fmt.Errorf("writing content: %w", err) } @@ -247,17 +115,6 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri } func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { - if localPath := b.localPath(path); localPath != "" { - r, err := os.Open(localPath) - if err != nil { - if os.IsNotExist(err) { - return nil, ErrNotFound - } - return nil, fmt.Errorf("opening local reader: %w", err) - } - return r, nil - } - r, err := b.bucket.NewReader(ctx, path, nil) if err != nil { if isNotExist(err) { @@ -268,11 +125,6 @@ func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { return r, nil } -// ReadsSeekable reports that Open returns seekable readers: an *os.File for -// file:// buckets and a *blob.Reader, which seeks with a ranged read, -// otherwise. -func (b *Blob) ReadsSeekable() bool { return true } - func (b *Blob) Exists(ctx context.Context, path string) (bool, error) { exists, err := b.bucket.Exists(ctx, path) if err != nil { @@ -281,20 +133,11 @@ func (b *Blob) Exists(ctx context.Context, path string) (bool, error) { return exists, nil } -// Delete removes the object at path. On a file:// bucket it also removes the -// object's fetch directory once empty, since fileblob leaves directories -// behind. Any other directory, such as a version directory another fetch may -// be creating its own directory in, is left alone. func (b *Blob) Delete(ctx context.Context, path string) error { err := b.bucket.Delete(ctx, path) if err != nil && !isNotExist(err) { return fmt.Errorf("deleting object: %w", err) } - if p := b.localPath(path); p != "" { - if dir := filepath.Dir(p); dir != b.fileRoot && isFetchDir(filepath.Base(dir)) { - _ = os.Remove(dir) // fails, harmlessly, while the directory holds anything - } - } return nil } diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 1a764de..d80290b 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -6,17 +6,11 @@ import ( "encoding/hex" "errors" "io" - "io/fs" - "os" "path/filepath" "runtime" "strings" - "sync" - "sync/atomic" "testing" "time" - - "gocloud.dev/blob" ) func TestOpenBucket(t *testing.T) { @@ -67,31 +61,16 @@ func TestBlobOpen(t *testing.T) { t.Fatalf("Open failed: %v", err) } defer func() { _ = r.Close() }() - seeker, ok := r.(io.Seeker) - if !ok { - t.Fatal("local file reader does not implement io.Seeker") - } - if _, err := seeker.Seek(9, io.SeekStart); err != nil { - t.Fatalf("Seek failed: %v", err) - } data, err := io.ReadAll(r) if err != nil { t.Fatalf("ReadAll failed: %v", err) } - if string(data) != "content" { - t.Errorf("content after seek = %q, want %q", string(data), "content") + if string(data) != content { + t.Errorf("content = %q, want %q", string(data), content) } } -// Blob.ReadsSeekable promises seekable readers for every bucket: file:// -// objects open as *os.File (checked in TestBlobOpen) and remote ones as -// *blob.Reader. -var ( - _ io.Seeker = (*blob.Reader)(nil) - _ SeekableReads = (*Blob)(nil) -) - func TestBlobOpenNotFound(t *testing.T) { b := createTestBlob(t) ctx := context.Background() @@ -142,44 +121,6 @@ func TestBlobDelete(t *testing.T) { } } -// A fetch stores its object in a directory of its own, so Delete removes that -// directory once it is empty rather than leave one behind per deleted object. -// A version directory from the layout before fetch directories is left alone, -// since a new fetch may be creating its directory inside it. -func TestBlobDeleteRemovesEmptyFetchDirectory(t *testing.T) { - dir := t.TempDir() - b := openFileBlob(t, dir) - ctx := context.Background() - const ( - emptied = "npm/pkg/1.0.0/0123456789abcdef/pkg.tgz" - shared = "npm/pkg/1.0.0/fedcba9876543210/pkg.tgz" - kept = "npm/pkg/1.0.0/fedcba9876543210/other.tgz" - legacy = "npm/pkg/2.0.0/pkg.tgz" - ) - for _, key := range []string{emptied, shared, kept, legacy} { - if _, _, err := b.Store(ctx, key, strings.NewReader("content")); err != nil { - t.Fatalf("Store(%q): %v", key, err) - } - } - - for _, key := range []string{emptied, shared, legacy} { - if err := b.Delete(ctx, key); err != nil { - t.Fatalf("Delete(%q): %v", key, err) - } - } - - pkg := filepath.Join(dir, "npm", "pkg") - if _, err := os.Stat(filepath.Join(pkg, "1.0.0", "0123456789abcdef")); !os.IsNotExist(err) { - t.Errorf("emptied fetch directory left behind (stat err %v)", err) - } - for _, d := range []string{filepath.Join("1.0.0", "fedcba9876543210"), "1.0.0", "2.0.0"} { - if _, err := os.Stat(filepath.Join(pkg, d)); err != nil { - t.Errorf("directory %s removed: %v", d, err) - } - } - assertReadsBack(t, b, kept, "content") -} - func TestBlobDeleteNotFound(t *testing.T) { b := createTestBlob(t) ctx := context.Background() @@ -337,11 +278,7 @@ func createTestBlob(t *testing.T) *Blob { t.Fatalf("OpenBucket failed: %v", err) } t.Cleanup(func() { _ = b.Close() }) - blob, ok := b.(*Blob) - if !ok { - t.Fatalf("OpenBucket returned %T, want *Blob", b) - } - return blob + return b } func fileURLFromPath(path string) string { @@ -352,357 +289,3 @@ func fileURLFromPath(path string) string { } return "file://" + path } - -func TestOpenBucketWritesNoAttrsSidecar(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - b, err := OpenBucket(ctx, fileURLFromPath(dir)) - if err != nil { - t.Fatalf("OpenBucket failed: %v", err) - } - defer func() { _ = b.Close() }() - - if _, _, err := b.Store(ctx, "pkg/thing-1.0.0.tgz", strings.NewReader("content")); err != nil { - t.Fatalf("Store failed: %v", err) - } - - sidecars, err := filepath.Glob(filepath.Join(dir, "*", "*.attrs")) - if err != nil { - t.Fatalf("Glob failed: %v", err) - } - if len(sidecars) != 0 { - t.Errorf("got sidecar files %v, want none: a truncated sidecar fails reads that overlap a write", sidecars) - } -} - -// A read overlapping a write to the same key must not fail. fileblob rewrote -// its ".attrs" sidecar in place, so a reader decoding it mid-write saw a -// partial file, which the proxy served as a 502 on an artifact it held. -func TestConcurrentReadsSurviveWritesToSameKey(t *testing.T) { - if runtime.GOOS == "windows" { - // Go opens files without FILE_SHARE_DELETE, so a writer cannot replace - // a file a reader holds open: its rename fails with access denied - // instead of contending. The other platforms exercise this race. - t.Skip("Windows refuses to replace a file readers hold open") - } - const ( - key = "pkg/thing-1.0.0.tgz" - readers = 4 - readsPerRead = 500 - ) - dir := t.TempDir() - ctx := context.Background() - - b, err := OpenBucket(ctx, fileURLFromPath(dir)) - if err != nil { - t.Fatalf("OpenBucket failed: %v", err) - } - defer func() { _ = b.Close() }() - - payload := strings.Repeat("x", 4096) - if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil { - t.Fatalf("seeding Store failed: %v", err) - } - - // The writer reports how it ended: a Store failure would otherwise stop - // the writes silently and let zero read failures pass for a test that - // never contended anything. - done := make(chan struct{}) - var writers sync.WaitGroup - var writes int - var writeErr error - writers.Add(1) - go func() { - defer writers.Done() - for { - select { - case <-done: - return - default: - } - if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil { - writeErr = err - return - } - writes++ - } - }() - - var failures atomic.Int64 - var reading sync.WaitGroup - for range readers { - reading.Add(1) - go func() { - defer reading.Done() - for range readsPerRead { - r, err := b.Open(ctx, key) - if err != nil { - failures.Add(1) - continue - } - if _, err := io.Copy(io.Discard, r); err != nil { - failures.Add(1) - } - _ = r.Close() - } - }() - } - reading.Wait() - close(done) - writers.Wait() - - if writeErr != nil { - t.Fatalf("writer stopped early: %v", writeErr) - } - if writes == 0 { - t.Fatal("no write completed, so the reads were never contended") - } - if got := failures.Load(); got != 0 { - t.Errorf("%d of %d reads failed while one writer rewrote the same key, want 0", got, readers*readsPerRead) - } -} - -// seedLegacySidecar stores key through a bucket that still writes sidecars, as -// an earlier version did, and returns the path fileblob actually used. It is -// discovered rather than assumed, so callers test the real mapping. -func seedLegacySidecar(t *testing.T, dir, key, payload string) string { - t.Helper() - ctx := context.Background() - - legacy, err := blob.OpenBucket(ctx, fileURLFromPath(dir)+"?no_tmp_dir=true") - if err != nil { - t.Fatalf("opening legacy bucket: %v", err) - } - if err := legacy.WriteAll(ctx, key, []byte(payload), nil); err != nil { - t.Fatalf("legacy WriteAll: %v", err) - } - if err := legacy.Close(); err != nil { - t.Fatalf("closing legacy bucket: %v", err) - } - - var found []string - walkErr := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if !d.IsDir() && strings.HasSuffix(path, ".attrs") { - found = append(found, path) - } - return nil - }) - if walkErr != nil { - t.Fatalf("walking %s: %v", dir, walkErr) - } - if len(found) != 1 { - t.Fatalf("got sidecars %v, want exactly one", found) - } - return found[0] -} - -// An interrupted setAttrs leaves a partial sidecar that fails every read of the -// key, and nothing rewrites one now, so a store has to clear it. -// -// One key per storage path the proxy builds: legacy artifact keys across ecosystems, -// metadata blobs, and the Gradle build cache. Scoped npm names, Go's "!" case -// escaping and the ":" in OCI digests and Debian epochs are the characters -// most likely to part fileblob's mapping from a plain path join. -func TestStoreClearsLegacyAttrsSidecar(t *testing.T) { - keys := []string{ - "npm/@babel/core/7.24.0/core-7.24.0.tgz", - "maven/org.apache.commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar", - "golang/github.com/!burnt!sushi/toml/v1.3.2/v1.3.2.zip", - "oci/library/nginx/sha256:abc123def456/manifest", - "debian/tzdata/1:2024a-1/tzdata_2024a-1_all.deb", - "pypi/requests/2.31.0/requests-2.31.0-py3-none-any.whl", - "cargo/serde/1.0.197/serde-1.0.197.crate", - "julia/Example/a1b2c3/a1b2c3.tar.gz", - "conda/numpy/1.26.4/numpy-1.26.4-py311.conda", - "_metadata/npm/@babel/core/metadata", - "_gradle/http-build-cache/0a1b2c3d4e5f", - // Keys fileblob escapes on every platform. - "npm/pkg//1.0.0/x.tgz", - "npm/pkg/../1.0.0/x.tgz", - } - - for _, key := range keys { - t.Run(key, func(t *testing.T) { - assertStoreClearsSidecar(t, key) - }) - } -} - -func assertStoreClearsSidecar(t *testing.T, key string) { - t.Helper() - const payload = "payload" - ctx := context.Background() - dir := t.TempDir() - - sidecar := seedLegacySidecar(t, dir, key, payload) - if err := os.WriteFile(sidecar, []byte(`{"user.content_type":"appl`), 0o600); err != nil { - t.Fatalf("corrupting sidecar: %v", err) - } - - b := openFileBlob(t, dir) - reader, err := b.bucket.NewReader(ctx, key, nil) - if err == nil { - _ = reader.Close() - t.Fatal("corrupt sidecar did not fail the read, so it is not the file fileblob reads for this key") - } - - derived := b.legacySidecarPath(key) - if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil { - t.Fatalf("Store failed: %v", err) - } - - if derived == "" { - t.Fatalf("legacySidecarPath declined %q, but fileblob wrote %q", key, sidecar) - } - if derived != sidecar { - t.Fatalf("derived %q, but fileblob wrote %q", derived, sidecar) - } - if _, err := os.Stat(sidecar); !os.IsNotExist(err) { - t.Errorf("sidecar still present after Store, stat err = %v", err) - } - assertReadsBack(t, b, key, payload) -} - -func assertReadsBack(t *testing.T, b *Blob, key, want string) { - t.Helper() - r, err := b.Open(context.Background(), key) - if err != nil { - t.Fatalf("read still failing after Store cleared the sidecar: %v", err) - } - defer func() { _ = r.Close() }() - got, err := io.ReadAll(r) - if err != nil { - t.Fatalf("ReadAll failed: %v", err) - } - if string(got) != want { - t.Errorf("got %q, want %q", got, want) - } -} - -func openFileBlob(t *testing.T, dir string) *Blob { - t.Helper() - s, err := OpenBucket(context.Background(), fileURLFromPath(dir)) - if err != nil { - t.Fatalf("OpenBucket failed: %v", err) - } - t.Cleanup(func() { _ = s.Close() }) - b, ok := s.(*Blob) - if !ok { - t.Fatalf("got %T, want *Blob", s) - } - return b -} - -// fileblob escapes a non-local key in a way this cannot reproduce, and one -// holding ".." resolves outside the cache directory. Removal declines both -// rather than delete the wrong file. -func TestLegacySidecarPathEscapesLikeFileblob(t *testing.T) { - root := filepath.FromSlash("/var/cache/proxy") - b := &Blob{fileRoot: root} - windows := runtime.GOOS == osWindows - - for _, tc := range []struct{ key, unix, windows string }{ - {"npm/pkg/1.0.0/x.tgz", "npm/pkg/1.0.0/x.tgz", "npm/pkg/1.0.0/x.tgz"}, - {"npm/pkg//1.0.0/x.tgz", "npm/pkg/__0x2f__1.0.0/x.tgz", "npm/pkg/__0x2f__1.0.0/x.tgz"}, - {"npm/pkg/../../etc/passwd", "npm/pkg/..__0x2f__..__0x2f__etc/passwd", "npm/pkg/..__0x2f__..__0x2f__etc/passwd"}, - {"npm/pkg/1.0.0/", "npm/pkg/1.0.0__0x2f__", "npm/pkg/1.0.0__0x2f__"}, - {"npm/a\x01b", "npm/a__0x1__b", "npm/a__0x1__b"}, - {"oci/nginx/sha256:abc/manifest", "oci/nginx/sha256:abc/manifest", "oci/nginx/sha256__0x3a__abc/manifest"}, - {"debian/tzdata/1:2024a-1/x.deb", "debian/tzdata/1:2024a-1/x.deb", "debian/tzdata/1__0x3a__2024a-1/x.deb"}, - {`npm/a\b`, `npm/a\b`, "npm/a__0x5c__b"}, - } { - want := tc.unix - if windows { - want = tc.windows - } - want = filepath.Join(root, filepath.FromSlash(want)) + attrsExt - if got := b.legacySidecarPath(tc.key); got != want { - t.Errorf("legacySidecarPath(%q) = %q, want %q", tc.key, got, want) - } - } -} - -func TestLegacySidecarPathDeclinesNonLocalKeys(t *testing.T) { - b := &Blob{fileRoot: filepath.FromSlash("/var/cache/proxy")} - - for _, key := range []string{"", ".", "..", "/etc/passwd"} { - if got := b.legacySidecarPath(key); got != "" { - t.Errorf("legacySidecarPath(%q) = %q, want \"\"", key, got) - } - } -} - -// Cloud backends have no local directory, so nothing is removed for them. -func TestLegacySidecarPathEmptyForCloudBackends(t *testing.T) { - b := &Blob{} - if got := b.legacySidecarPath("npm/pkg/1.0.0/x.tgz"); got != "" { - t.Errorf("legacySidecarPath = %q, want \"\" when there is no file root", got) - } -} - -// Cleanup that cannot complete must not fail the write. A non-empty directory -// at the sidecar path makes os.Remove fail with something other than not-exist -// on every platform, which is what a Windows sharing violation would look like -// here. -func TestStoreSucceedsWhenSidecarCannotBeRemoved(t *testing.T) { - const key = "npm/pkg/1.0.0/pkg-1.0.0.tgz" - const payload = "payload" - dir := t.TempDir() - ctx := context.Background() - - b := openFileBlob(t, dir) - - sidecar := filepath.Join(dir, filepath.FromSlash(key)) + ".attrs" - if err := os.MkdirAll(filepath.Join(sidecar, "blocker"), 0o750); err != nil { - t.Fatalf("seeding an unremovable sidecar: %v", err) - } - if err := os.Remove(sidecar); err == nil { - t.Fatal("sidecar path was removable, so the test proves nothing") - } - - if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil { - t.Errorf("Store failed because cleanup could not complete: %v", err) - } -} - -type failingReader struct { - data []byte - err error -} - -func (r *failingReader) Read(p []byte) (int, error) { - if len(r.data) == 0 { - return 0, r.err - } - n := copy(p, r.data) - r.data = r.data[n:] - return n, nil -} - -// A download that breaks off midway must not leave its first half behind as -// an object, or every failed fetch adds an orphan nothing ever deletes. -func TestBlobStoreFailedCopyLeavesNoObject(t *testing.T) { - b := createTestBlob(t) - ctx := context.Background() - readErr := errors.New("connection reset") - - _, _, err := b.Store(ctx, "npm/lodash/4.17.21/lodash.tgz", &failingReader{ - data: []byte(strings.Repeat("partial", 1024)), - err: readErr, - }) - if !errors.Is(err, readErr) { - t.Fatalf("Store error = %v, want %v", err, readErr) - } - - exists, err := b.Exists(ctx, "npm/lodash/4.17.21/lodash.tgz") - if err != nil { - t.Fatalf("Exists failed: %v", err) - } - if exists { - t.Error("failed Store left a partial object behind") - } -} diff --git a/internal/storage/filesystem.go b/internal/storage/filesystem.go new file mode 100644 index 0000000..1e5a24f --- /dev/null +++ b/internal/storage/filesystem.go @@ -0,0 +1,257 @@ +package storage + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + fsys "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +// Filesystem implements Storage using the local filesystem. +type Filesystem struct { + root string +} + +// NewFilesystem creates a new filesystem storage rooted at the given directory. +// The directory will be created if it does not exist. +func NewFilesystem(root string) (*Filesystem, error) { + absRoot, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("resolving root path: %w", err) + } + + if err := os.MkdirAll(absRoot, dirPermissions); err != nil { + return nil, fmt.Errorf("creating root directory: %w", err) + } + + return &Filesystem{root: absRoot}, nil +} + +func (fs *Filesystem) fullPath(path string) (string, error) { + full := filepath.Clean(filepath.Join(fs.root, filepath.FromSlash(path))) + if full != fs.root && !strings.HasPrefix(full, fs.root+string(filepath.Separator)) { + return "", fmt.Errorf("%w: path escapes storage root", ErrNotFound) + } + return full, nil +} + +func (fs *Filesystem) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { + fullPath, err := fs.fullPath(path) + if err != nil { + return 0, "", err + } + + dir := filepath.Dir(fullPath) + if err := os.MkdirAll(dir, dirPermissions); err != nil { + return 0, "", fmt.Errorf("creating directory: %w", err) + } + + // Write to temp file first for atomic operation + tmpFile, err := os.CreateTemp(dir, ".tmp-*") + if err != nil { + return 0, "", fmt.Errorf("creating temp file: %w", err) + } + tmpPath := tmpFile.Name() + + // Clean up temp file on error + success := false + defer func() { + if !success { + _ = tmpFile.Close() + _ = os.Remove(tmpPath) + } + }() + + // Write content and compute hash + h := sha256.New() + w := io.MultiWriter(tmpFile, h) + + size, err := io.Copy(w, r) + if err != nil { + return 0, "", fmt.Errorf("writing content: %w", err) + } + + if err := tmpFile.Close(); err != nil { + return 0, "", fmt.Errorf("closing temp file: %w", err) + } + + // Atomic rename + if err := os.Rename(tmpPath, fullPath); err != nil { + return 0, "", fmt.Errorf("renaming temp file: %w", err) + } + + success = true + hash := hex.EncodeToString(h.Sum(nil)) + return size, hash, nil +} + +func (fs *Filesystem) Open(ctx context.Context, path string) (io.ReadCloser, error) { + fullPath, err := fs.fullPath(path) + if err != nil { + return nil, err + } + + f, err := os.Open(fullPath) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("opening file: %w", err) + } + + return f, nil +} + +func (fs *Filesystem) Exists(ctx context.Context, path string) (bool, error) { + fullPath, err := fs.fullPath(path) + if err != nil { + return false, err + } + + _, err = os.Stat(fullPath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("checking file: %w", err) + } + + return true, nil +} + +func (fs *Filesystem) Delete(ctx context.Context, path string) error { + fullPath, err := fs.fullPath(path) + if err != nil { + return err + } + + err = os.Remove(fullPath) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("removing file: %w", err) + } + + // Try to clean up empty parent directories + dir := filepath.Dir(fullPath) + for dir != fs.root { + if err := os.Remove(dir); err != nil { + break // Directory not empty or other error + } + dir = filepath.Dir(dir) + } + + return nil +} + +func (fs *Filesystem) SignedURL(_ context.Context, _ string, _ time.Duration) (string, error) { + return "", ErrSignedURLUnsupported +} + +func (fs *Filesystem) Size(ctx context.Context, path string) (int64, error) { + fullPath, err := fs.fullPath(path) + if err != nil { + return 0, err + } + + info, err := os.Stat(fullPath) + if err != nil { + if os.IsNotExist(err) { + return 0, ErrNotFound + } + return 0, fmt.Errorf("stat file: %w", err) + } + + return info.Size(), nil +} + +func (fs *Filesystem) UsedSpace(ctx context.Context) (int64, error) { + var total int64 + + err := filepath.Walk(fs.root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + total += info.Size() + } + return nil + }) + if err != nil { + return 0, fmt.Errorf("walking directory: %w", err) + } + + return total, nil +} + +// ListPrefix returns object metadata for paths under a prefix. +func (fs *Filesystem) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) { + searchRoot, err := fs.fullPath(prefix) + if err != nil { + return nil, err + } + + if _, err := os.Stat(searchRoot); err != nil { + if os.IsNotExist(err) { + return []ObjectInfo{}, nil + } + return nil, fmt.Errorf("stat prefix: %w", err) + } + + objects := make([]ObjectInfo, 0) + err = filepath.WalkDir(searchRoot, func(path string, entry fsys.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + + info, err := entry.Info() + if err != nil { + return err + } + + relPath, err := filepath.Rel(fs.root, path) + if err != nil { + return err + } + + objects = append(objects, ObjectInfo{ + Path: filepath.ToSlash(relPath), + Size: info.Size(), + ModTime: info.ModTime(), + }) + + return nil + }) + if err != nil { + return nil, fmt.Errorf("walking prefix: %w", err) + } + + return objects, nil +} + +// Root returns the root directory of the storage. +func (fs *Filesystem) Root() string { + return fs.root +} + +// FullPath returns the full filesystem path for a storage path. +// Useful for serving files directly or debugging. +// Returns an error if the resulting path would escape the storage root. +func (fs *Filesystem) FullPath(path string) (string, error) { + return fs.fullPath(path) +} + +func (fs *Filesystem) URL() string { + return "file://" + filepath.ToSlash(fs.root) +} + +func (fs *Filesystem) Close() error { + return nil +} diff --git a/internal/storage/filesystem_test.go b/internal/storage/filesystem_test.go new file mode 100644 index 0000000..332dfbf --- /dev/null +++ b/internal/storage/filesystem_test.go @@ -0,0 +1,280 @@ +package storage + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestNewFilesystem(t *testing.T) { + dir := t.TempDir() + root := filepath.Join(dir, "cache") + + fs, err := NewFilesystem(root) + if err != nil { + t.Fatalf("NewFilesystem failed: %v", err) + } + + if _, err := os.Stat(root); err != nil { + t.Errorf("root directory not created: %v", err) + } + + if fs.Root() != root { + t.Errorf("Root() = %q, want %q", fs.Root(), root) + } +} + +func TestFilesystemStore(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + content := "test content for storage" + + size, hash, err := fs.Store(ctx, "npm/lodash/4.17.21/lodash.tgz", strings.NewReader(content)) + if err != nil { + t.Fatalf("Store failed: %v", err) + } + + if size != int64(len(content)) { + t.Errorf("size = %d, want %d", size, len(content)) + } + + h := sha256.Sum256([]byte(content)) + wantHash := hex.EncodeToString(h[:]) + if hash != wantHash { + t.Errorf("hash = %s, want %s", hash, wantHash) + } + + // Verify file exists on disk + fullPath, err := fs.FullPath("npm/lodash/4.17.21/lodash.tgz") + if err != nil { + t.Fatalf("FullPath failed: %v", err) + } + data, err := os.ReadFile(fullPath) + if err != nil { + t.Fatalf("reading stored file: %v", err) + } + if string(data) != content { + t.Errorf("stored content = %q, want %q", string(data), content) + } +} + +func TestFilesystemStoreAtomic(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + + // Store initial content + _, _, err := fs.Store(ctx, "test/file.txt", strings.NewReader("initial")) + if err != nil { + t.Fatalf("initial Store failed: %v", err) + } + + // Overwrite with new content + _, _, err = fs.Store(ctx, "test/file.txt", strings.NewReader("updated")) + if err != nil { + t.Fatalf("update Store failed: %v", err) + } + + // Verify updated content + r, err := fs.Open(ctx, "test/file.txt") + if err != nil { + t.Fatalf("Open failed: %v", err) + } + defer func() { _ = r.Close() }() + + data, _ := io.ReadAll(r) + if string(data) != "updated" { + t.Errorf("content = %q, want %q", string(data), "updated") + } +} + +func TestFilesystemOpen(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + content := "readable content" + + _, _, _ = fs.Store(ctx, "test/read.txt", strings.NewReader(content)) + + r, err := fs.Open(ctx, "test/read.txt") + if err != nil { + t.Fatalf("Open failed: %v", err) + } + defer func() { _ = r.Close() }() + + data, err := io.ReadAll(r) + if err != nil { + t.Fatalf("ReadAll failed: %v", err) + } + if string(data) != content { + t.Errorf("content = %q, want %q", string(data), content) + } +} + +func TestFilesystemOpenNotFound(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + + _, err := fs.Open(ctx, "does/not/exist.txt") + if !errors.Is(err, ErrNotFound) { + t.Errorf("Open non-existent = %v, want ErrNotFound", err) + } +} + +func TestFilesystemExists(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + + exists, err := fs.Exists(ctx, "test/exists.txt") + if err != nil { + t.Fatalf("Exists failed: %v", err) + } + if exists { + t.Error("Exists returned true for non-existent file") + } + + _, _, _ = fs.Store(ctx, "test/exists.txt", strings.NewReader("content")) + + exists, err = fs.Exists(ctx, "test/exists.txt") + if err != nil { + t.Fatalf("Exists after store failed: %v", err) + } + if !exists { + t.Error("Exists returned false for existing file") + } +} + +func TestFilesystemDelete(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + + _, _, _ = fs.Store(ctx, "test/delete/nested/file.txt", strings.NewReader("content")) + + err := fs.Delete(ctx, "test/delete/nested/file.txt") + if err != nil { + t.Fatalf("Delete failed: %v", err) + } + + exists, _ := fs.Exists(ctx, "test/delete/nested/file.txt") + if exists { + t.Error("file still exists after delete") + } + + // Empty parent directories should be cleaned up + nestedDir, err := fs.FullPath("test/delete/nested") + if err != nil { + t.Fatalf("FullPath failed: %v", err) + } + if _, err := os.Stat(nestedDir); !os.IsNotExist(err) { + t.Error("empty nested directory not cleaned up") + } +} + +func TestFilesystemDeleteNotFound(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + + // Delete non-existent file should not error + err := fs.Delete(ctx, "does/not/exist.txt") + if err != nil { + t.Errorf("Delete non-existent = %v, want nil", err) + } +} + +func TestFilesystemSize(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + content := "size test content" + + _, _, _ = fs.Store(ctx, "test/size.txt", strings.NewReader(content)) + + size, err := fs.Size(ctx, "test/size.txt") + if err != nil { + t.Fatalf("Size failed: %v", err) + } + if size != int64(len(content)) { + t.Errorf("Size = %d, want %d", size, len(content)) + } +} + +func TestFilesystemSizeNotFound(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + + _, err := fs.Size(ctx, "does/not/exist.txt") + if !errors.Is(err, ErrNotFound) { + t.Errorf("Size non-existent = %v, want ErrNotFound", err) + } +} + +func TestFilesystemUsedSpace(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + + // Empty storage + used, err := fs.UsedSpace(ctx) + if err != nil { + t.Fatalf("UsedSpace failed: %v", err) + } + if used != 0 { + t.Errorf("UsedSpace empty = %d, want 0", used) + } + + // Add some files + _, _, _ = fs.Store(ctx, "a.txt", strings.NewReader("aaaa")) // 4 bytes + _, _, _ = fs.Store(ctx, "b.txt", strings.NewReader("bbbbbb")) // 6 bytes + _, _, _ = fs.Store(ctx, "c/d.txt", strings.NewReader("ccccc")) // 5 bytes + + used, err = fs.UsedSpace(ctx) + if err != nil { + t.Fatalf("UsedSpace failed: %v", err) + } + if used != 15 { + t.Errorf("UsedSpace = %d, want 15", used) + } +} + +func TestFilesystemLargeFile(t *testing.T) { + assertLargeFileRoundTrip(t, createTestFilesystem(t)) +} + +func TestFilesystemRejectsTraversal(t *testing.T) { + tmp := t.TempDir() + fs, err := NewFilesystem(tmp) + if err != nil { + t.Fatal(err) + } + for _, p := range []string{"../etc/passwd", "../../etc/passwd", "a/../../etc/passwd"} { + if _, err := fs.Open(context.Background(), p); err == nil { + t.Errorf("Open(%q) should reject traversal", p) + } + if _, _, err := fs.Store(context.Background(), p, strings.NewReader("x")); err == nil { + t.Errorf("Store(%q) should reject traversal", p) + } + } +} + +func TestFilesystemSignedURLUnsupported(t *testing.T) { + fs := createTestFilesystem(t) + + _, err := fs.SignedURL(context.Background(), "test/file.txt", time.Minute) + if !errors.Is(err, ErrSignedURLUnsupported) { + t.Errorf("SignedURL = %v, want ErrSignedURLUnsupported", err) + } +} + +func createTestFilesystem(t *testing.T) *Filesystem { + t.Helper() + dir := t.TempDir() + + fs, err := NewFilesystem(dir) + if err != nil { + t.Fatalf("NewFilesystem failed: %v", err) + } + return fs +} diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go deleted file mode 100644 index a096c18..0000000 --- a/internal/storage/gcs.go +++ /dev/null @@ -1,97 +0,0 @@ -package storage - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "io" - "time" - - gcstorage "github.com/git-pkgs/gcs" -) - -// GCS adapts a Google Cloud Storage bucket to Storage. -type GCS struct { - bucket *gcstorage.Bucket - url string -} - -// OpenGCS opens a Google Cloud Storage bucket from a gs:// URL. -func OpenGCS(ctx context.Context, urlStr string) (*GCS, error) { - bucket, err := gcstorage.OpenBucket(ctx, urlStr) - if err != nil { - return nil, err - } - return &GCS{bucket: bucket, url: urlStr}, nil -} - -func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { - h := sha256.New() - size, err := g.bucket.Write(ctx, path, io.TeeReader(r, h)) - if err != nil { - return 0, "", err - } - return size, hex.EncodeToString(h.Sum(nil)), nil -} - -func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) { - r, err := g.bucket.Open(ctx, path) - if errors.Is(err, gcstorage.ErrNotFound) { - return nil, ErrNotFound - } - return r, err -} - -func (g *GCS) Exists(ctx context.Context, path string) (bool, error) { - return g.bucket.Exists(ctx, path) -} - -func (g *GCS) Delete(ctx context.Context, path string) error { - return g.bucket.Delete(ctx, path) -} - -func (g *GCS) Size(ctx context.Context, path string) (int64, error) { - size, err := g.bucket.Size(ctx, path) - if errors.Is(err, gcstorage.ErrNotFound) { - return 0, ErrNotFound - } - return size, err -} - -func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { - u, err := g.bucket.SignedURL(ctx, path, expiry) - if errors.Is(err, gcstorage.ErrSignedURLUnsupported) { - return "", ErrSignedURLUnsupported - } - return u, err -} - -func (g *GCS) UsedSpace(ctx context.Context) (int64, error) { - return g.bucket.UsedSpace(ctx) -} - -func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) { - objects, err := g.bucket.ListPrefix(ctx, prefix) - if err != nil { - return nil, err - } - - result := make([]ObjectInfo, 0, len(objects)) - for _, object := range objects { - result = append(result, ObjectInfo{ - Path: object.Name, - Size: object.Size, - ModTime: object.ModTime, - }) - } - return result, nil -} - -func (g *GCS) Close() error { - return nil -} - -func (g *GCS) URL() string { - return g.url -} diff --git a/internal/storage/gcs_test.go b/internal/storage/gcs_test.go deleted file mode 100644 index 1bd8a40..0000000 --- a/internal/storage/gcs_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package storage - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "io" - "net/http" - "net/http/httptest" - "net/url" - "sort" - "strconv" - "strings" - "testing" - "time" -) - -func TestOpenBucketGCSRoundTripWithEmulator(t *testing.T) { - server := httptest.NewServer(&fakeGCSServer{t: t, objects: map[string]string{}}) - defer server.Close() - t.Setenv("STORAGE_EMULATOR_HOST", server.URL) - - ctx := context.Background() - store, err := OpenBucket(ctx, "gs://test-bucket") - if err != nil { - t.Fatalf("OpenBucket failed: %v", err) - } - - size, hash, err := store.Store(ctx, "npm/pkg/file.tgz", strings.NewReader("content")) - if err != nil { - t.Fatalf("Store failed: %v", err) - } - wantHash := sha256.Sum256([]byte("content")) - if size != int64(len("content")) || hash != hex.EncodeToString(wantHash[:]) { - t.Fatalf("Store returned size=%d hash=%q", size, hash) - } - - exists, err := store.Exists(ctx, "npm/pkg/file.tgz") - if err != nil || !exists { - t.Fatalf("Exists = %v, %v; want true, nil", exists, err) - } - - r, err := store.Open(ctx, "npm/pkg/file.tgz") - if err != nil { - t.Fatalf("Open failed: %v", err) - } - data, _ := io.ReadAll(r) - _ = r.Close() - if string(data) != "content" { - t.Fatalf("Open content = %q, want content", data) - } - - lister, ok := store.(interface { - ListPrefix(context.Context, string) ([]ObjectInfo, error) - }) - if !ok { - t.Fatal("GCS storage does not support prefix listing") - } - list, err := lister.ListPrefix(ctx, "npm/") - if err != nil { - t.Fatalf("ListPrefix failed: %v", err) - } - if len(list) != 1 || list[0].Path != "npm/pkg/file.tgz" { - t.Fatalf("ListPrefix = %#v", list) - } - - if err := store.Delete(ctx, "npm/pkg/file.tgz"); err != nil { - t.Fatalf("Delete failed: %v", err) - } - exists, err = store.Exists(ctx, "npm/pkg/file.tgz") - if err != nil || exists { - t.Fatalf("Exists after delete = %v, %v; want false, nil", exists, err) - } - if err := store.Delete(ctx, "npm/pkg/file.tgz"); err != nil { - t.Fatalf("Delete missing object = %v, want nil", err) - } - - reader, err := store.Open(ctx, "npm/pkg/file.tgz") - if reader != nil || !errors.Is(err, ErrNotFound) { - t.Fatalf("Open missing object = %v, %v; want nil, ErrNotFound", reader, err) - } - if _, err := store.Size(ctx, "npm/pkg/file.tgz"); !errors.Is(err, ErrNotFound) { - t.Fatalf("Size missing object = %v, want ErrNotFound", err) - } - if _, err := store.SignedURL(ctx, "npm/pkg/file.tgz", time.Minute); !errors.Is(err, ErrSignedURLUnsupported) { - t.Fatalf("SignedURL with emulator = %v, want ErrSignedURLUnsupported", err) - } -} - -type fakeGCSServer struct { - t *testing.T - objects map[string]string -} - -func (f *fakeGCSServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { - switch { - case r.Method == http.MethodPost && r.URL.Path == "/upload/storage/v1/b/test-bucket/o": - name := r.URL.Query().Get("name") - data, _ := io.ReadAll(r.Body) - f.objects[name] = string(data) - writeJSON(w, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)}) - case r.Method == http.MethodGet && r.URL.Path == "/storage/v1/b/test-bucket/o": - prefix := r.URL.Query().Get("prefix") - page := fakeGCSListResponse{} - for name, data := range f.objects { - if strings.HasPrefix(name, prefix) { - page.Items = append(page.Items, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)}) - } - } - sort.Slice(page.Items, func(i, j int) bool { return page.Items[i].Name < page.Items[j].Name }) - writeJSON(w, page) - case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"): - name := objectNameFromPath(r.URL.Path) - data, ok := f.objects[name] - if !ok { - http.NotFound(w, r) - return - } - if r.URL.Query().Get("alt") == "media" { - _, _ = io.WriteString(w, data) - return - } - writeJSON(w, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)}) - case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"): - name := objectNameFromPath(r.URL.Path) - if _, ok := f.objects[name]; !ok { - http.NotFound(w, r) - return - } - delete(f.objects, name) - w.WriteHeader(http.StatusNoContent) - default: - f.t.Errorf("unexpected request: %s %s", r.Method, r.URL.String()) - http.Error(w, "unexpected request", http.StatusInternalServerError) - } -} - -type fakeGCSObject struct { - Name string `json:"name"` - Size string `json:"size"` - Updated string `json:"updated"` -} - -type fakeGCSListResponse struct { - NextPageToken string `json:"nextPageToken"` - Items []fakeGCSObject `json:"items"` -} - -func writeJSON(w http.ResponseWriter, v any) { - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(v) -} - -func objectNameFromPath(p string) string { - escaped := strings.TrimPrefix(p, "/storage/v1/b/test-bucket/o/") - name, _ := url.PathUnescape(escaped) - return name -} diff --git a/internal/storage/s3_upload_test.go b/internal/storage/s3_upload_test.go deleted file mode 100644 index 898b1f7..0000000 --- a/internal/storage/s3_upload_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package storage - -import ( - "bytes" - "context" - "fmt" - "io" - "maps" - "net/http" - "net/http/httptest" - "slices" - "strconv" - "strings" - "sync" - "testing" -) - -// fakeS3 implements just enough of the S3 API for PutObject and multipart -// uploads, recording the size of each part it receives. -type fakeS3 struct { - mu sync.Mutex - objects map[string][]byte - parts map[int][]byte - partSizes []int - // discard drops bodies so a benchmark measures only the client. - discard bool -} - -func newFakeS3(t testing.TB) (*fakeS3, string) { - t.Helper() - f := &fakeS3{objects: map[string][]byte{}, parts: map[int][]byte{}} - server := httptest.NewServer(f) - t.Cleanup(server.Close) - url := "s3://bucket?endpoint=" + server.URL + "®ion=us-east-1&use_path_style=true&anonymous=true&request_checksum_calculation=when_required" - return f, url -} - -func (f *fakeS3) ServeHTTP(w http.ResponseWriter, r *http.Request) { - key := strings.TrimPrefix(r.URL.Path, "/bucket/") - q := r.URL.Query() - if f.discard { - _, _ = io.Copy(io.Discard, r.Body) - r.Body = http.NoBody - } - body, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - f.mu.Lock() - defer f.mu.Unlock() - switch { - case r.Method == http.MethodPost && q.Has("uploads"): - f.parts = map[int][]byte{} - _, _ = fmt.Fprintf(w, `bucket%supload`, key) - case r.Method == http.MethodPut && q.Has("partNumber"): - n, _ := strconv.Atoi(q.Get("partNumber")) - f.parts[n] = body - f.partSizes = append(f.partSizes, len(body)) - w.Header().Set("ETag", fmt.Sprintf(`"part%d"`, n)) - case r.Method == http.MethodPost && q.Has("uploadId"): - var object []byte - for _, n := range slices.Sorted(maps.Keys(f.parts)) { - object = append(object, f.parts[n]...) - } - f.objects[key] = object - _, _ = fmt.Fprintf(w, `bucket%s"object"`, key) - case r.Method == http.MethodPut: - f.objects[key] = body - w.Header().Set("ETag", `"object"`) - default: - http.Error(w, "unsupported", http.StatusNotImplemented) - } -} - -func TestS3StoreUsesSmallUploadParts(t *testing.T) { - fake, url := newFakeS3(t) - store, err := OpenBucket(context.Background(), url) - if err != nil { - t.Fatal(err) - } - defer func() { _ = store.Close() }() - - small := []byte("small object") - large := bytes.Repeat([]byte("0123456789abcdef"), 20<<20/16) - for name, data := range map[string][]byte{"small": small, "large": large} { - size, _, err := store.Store(context.Background(), name, bytes.NewReader(data)) - if err != nil { - t.Fatalf("Store(%s): %v", name, err) - } - if size != int64(len(data)) || !bytes.Equal(fake.objects[name], data) { - t.Errorf("Store(%s) wrote %d bytes, stored %d; want %d", name, size, len(fake.objects[name]), len(data)) - } - } - if len(fake.partSizes) == 0 { - t.Fatal("large object was not uploaded in parts") - } - for _, n := range fake.partSizes { - if n > uploadBufferSize { - t.Errorf("part of %d bytes, want at most %d", n, uploadBufferSize) - } - } -} - -func BenchmarkS3Store(b *testing.B) { - fake, url := newFakeS3(b) - fake.discard = true - store, err := OpenBucket(context.Background(), url) - if err != nil { - b.Fatal(err) - } - defer func() { _ = store.Close() }() - data := bytes.Repeat([]byte("0123456789abcdef"), 32<<20/16) - b.SetBytes(int64(len(data))) - for b.Loop() { - if _, _, err := store.Store(context.Background(), "object", bytes.NewReader(data)); err != nil { - b.Fatal(err) - } - } -} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index edca684..e11db53 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -5,16 +5,13 @@ // - file:///path/to/dir - Local filesystem storage // - s3://bucket-name - Amazon S3 // - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO) -// - gs://bucket-name - Google Cloud Storage (supports GKE Workload Identity -// via Application Default Credentials) -// - azblob://container-name - Azure Blob Storage // // Use OpenBucket to create a storage backend from a URL. package storage import ( "context" - "crypto/rand" + "crypto/sha256" "encoding/hex" "errors" "io" @@ -46,9 +43,6 @@ type Storage interface { // Open returns a reader for the content at path. // The caller must close the reader when done. - // The reader may also implement io.Seeker when seeking is efficient for the - // storage backend. Backends must not expose io.Seeker when seeking requires - // reading and discarding the bytes before the requested position. // Returns ErrNotFound if the path does not exist. Open(ctx context.Context, path string) (io.ReadCloser, error) @@ -77,42 +71,51 @@ type Storage interface { Close() error } -// SeekableReads is implemented by backends whose Open always returns a -// reader that implements io.Seeker. It tells a caller that byte ranges can be -// served from an object without opening it first. -type SeekableReads interface { - ReadsSeekable() bool -} - -// FetchPath builds the storage path for one fetch of an artifact: -// {ecosystem}/{name}/{version}/{fetchID}/{filename}. Each fetch writes its own -// object, so no fetch overwrites or deletes another's. -func FetchPath(ecosystem, name, version, fetchID, filename string) string { - return ecosystem + "/" + name + "/" + version + "/" + fetchID + "/" + filename -} - -// fetchIDBytes sizes a fetch id: 16 hex characters keeps paths short, which -// matters on Windows, and a collision between fetches of one artifact version -// is out of reach. -const fetchIDBytes = 8 - -// NewFetchID returns a random id for FetchPath. -func NewFetchID() string { - b := make([]byte, fetchIDBytes) - _, _ = rand.Read(b) - return hex.EncodeToString(b) -} - -// isFetchDir reports whether a directory name has the shape of a fetch id. -// Only one fetch ever writes into such a directory. -func isFetchDir(name string) bool { - if len(name) != 2*fetchIDBytes { - return false +// ArtifactPath builds a storage path for an artifact. +// Format: {ecosystem}/{namespace}/{name}/{version}/{filename} +// For packages without namespace: {ecosystem}/{name}/{version}/{filename} +func ArtifactPath(ecosystem, namespace, name, version, filename string) string { + if namespace != "" { + return ecosystem + "/" + namespace + "/" + name + "/" + version + "/" + filename } - for _, c := range name { - if (c < '0' || c > '9') && (c < 'a' || c > 'f') { - return false - } - } - return true + return ecosystem + "/" + name + "/" + version + "/" + filename +} + +// HashingReader wraps a reader and computes SHA256 hash as content is read. +type HashingReader struct { + r io.Reader + hash []byte + h interface{ Sum([]byte) []byte } + size int64 + done bool +} + +func NewHashingReader(r io.Reader) *HashingReader { + h := sha256.New() + return &HashingReader{ + r: io.TeeReader(r, h), + h: h, + } +} + +func (hr *HashingReader) Read(p []byte) (n int, err error) { + n, err = hr.r.Read(p) + hr.size += int64(n) + if err == io.EOF { + hr.done = true + hr.hash = hr.h.Sum(nil) + } + return +} + +func (hr *HashingReader) Sum() string { + if !hr.done { + hr.hash = hr.h.Sum(nil) + hr.done = true + } + return hex.EncodeToString(hr.hash) +} + +func (hr *HashingReader) Size() int64 { + return hr.size } diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 5d4f0b4..97800f0 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -6,19 +6,56 @@ import ( "crypto/sha256" "encoding/hex" "io" + "strings" "testing" ) -func TestIsFetchDir(t *testing.T) { - for range 10 { - if id := NewFetchID(); !isFetchDir(id) { - t.Errorf("isFetchDir(%q) = false for a fetch id", id) +func TestArtifactPath(t *testing.T) { + tests := []struct { + ecosystem string + namespace string + name string + version string + filename string + want string + }{ + {"npm", "", "lodash", "4.17.21", "lodash-4.17.21.tgz", "npm/lodash/4.17.21/lodash-4.17.21.tgz"}, + {"npm", "babel", "core", "7.0.0", "core-7.0.0.tgz", "npm/babel/core/7.0.0/core-7.0.0.tgz"}, + {"cargo", "", "serde", "1.0.0", "serde-1.0.0.crate", "cargo/serde/1.0.0/serde-1.0.0.crate"}, + {"pypi", "", "requests", "2.28.0", "requests-2.28.0.tar.gz", "pypi/requests/2.28.0/requests-2.28.0.tar.gz"}, + {"maven", "org.apache", "commons-lang3", "3.12.0", "commons-lang3-3.12.0.jar", "maven/org.apache/commons-lang3/3.12.0/commons-lang3-3.12.0.jar"}, + } + + for _, tt := range tests { + got := ArtifactPath(tt.ecosystem, tt.namespace, tt.name, tt.version, tt.filename) + if got != tt.want { + t.Errorf("ArtifactPath(%q, %q, %q, %q, %q) = %q, want %q", + tt.ecosystem, tt.namespace, tt.name, tt.version, tt.filename, got, tt.want) } } - for _, name := range []string{"1.0.0", "0123456789abcde", "0123456789abcdef0", "0123456789ABCDEF", "0123456789abcdeg", ""} { - if isFetchDir(name) { - t.Errorf("isFetchDir(%q) = true", name) - } +} + +func TestHashingReader(t *testing.T) { + content := "hello world" + r := NewHashingReader(strings.NewReader(content)) + + data, err := io.ReadAll(r) + if err != nil { + t.Fatalf("ReadAll failed: %v", err) + } + + if string(data) != content { + t.Errorf("got content %q, want %q", string(data), content) + } + + if r.Size() != int64(len(content)) { + t.Errorf("got size %d, want %d", r.Size(), len(content)) + } + + h := sha256.Sum256([]byte(content)) + wantHash := hex.EncodeToString(h[:]) + if r.Sum() != wantHash { + t.Errorf("got hash %s, want %s", r.Sum(), wantHash) } } diff --git a/tools.go b/tools.go new file mode 100644 index 0000000..caf60b1 --- /dev/null +++ b/tools.go @@ -0,0 +1,7 @@ +//go:build tools + +package tools + +import ( + _ "github.com/swaggo/swag/cmd/swag" +)