diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..05fb224 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +# Version and commit come in as build args, so the build doesn't need .git. +.git +.github + +# Local build output and runtime data +/proxy +dist/ +build/ +cache/ +*.db +*.tar.gz + +# Local config and secrets +.env +.env.local +proxy.cfg +config.yaml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0fadcbf..e325a8a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Log in to the Container registry - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 with: registry: ghcr.io username: ${{ github.actor }} @@ -61,6 +61,9 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + COMMIT=${{ github.sha }} provenance: mode=max sbom: true diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 02a0722..00041a6 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -34,4 +34,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e # v0.7.1 + uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0 diff --git a/Dockerfile b/Dockerfile index 1a934ed..d434e58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,9 +14,13 @@ COPY . . # Build the binary for the target platform ARG TARGETARCH -RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /proxy ./cmd/proxy +ARG VERSION=dev +ARG COMMIT=unknown +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ + -ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${COMMIT}" \ + -o /proxy ./cmd/proxy -FROM alpine:3.24.2 +FROM alpine:3.24.3 RUN apk add --no-cache ca-certificates git @@ -31,5 +35,9 @@ WORKDIR /data EXPOSE 8080 +# /health checks the database and storage and can take up to ten seconds. +HEALTHCHECK --interval=30s --timeout=11s --start-period=10s --retries=3 \ + CMD wget -q -O /dev/null http://127.0.0.1:8080/health || exit 1 + ENTRYPOINT ["proxy"] CMD ["serve"] diff --git a/README.md b/README.md index a324c53..dff6964 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ A 3-day cooldown means that when `lodash` publishes version `4.18.0`, your build Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default and carve out exceptions for packages where you need faster updates. See [docs/configuration.md](docs/configuration.md) for the full config reference. +One instance can offer several cooldown policies through [named profiles](docs/configuration.md#cooldown-profiles). Configure `cooldown.profiles` and point clients at paths such as `/cooldown/3d/npm/` or `/cooldown/7d/cargo/`. Profiles share the package cache; existing URLs keep using the global policy. + ## Artifact Scanning Cooldown only looks at a version's publish timestamp — it never inspects the actual bytes. Artifact scanning closes that gap: when enabled, every artifact is staged into storage and scanned by one or more external services (trivy, ClamAV, Wiz, or anything else that speaks a small HTTP/JSON contract) before it's committed to the cache and served to clients. @@ -75,7 +77,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi Cooldown requires publish timestamps in metadata. Registries without a "Yes" in the cooldown column either don't expose timestamps or haven't been wired up yet. -\* Hex cooldown requires disabling registry signature verification (`HEX_NO_VERIFY_REPO_ORIGIN=1`) since the proxy re-encodes the protobuf payload. +\* Hex cooldown requires disabling registry signature verification (`HEX_UNSAFE_REGISTRY=1`) since the proxy re-encodes the protobuf payload. Origin verification can stay enabled. † Go cooldown uses the time in each version's `.info`, which is the commit time of the tagged revision, not when the tag was pushed. @@ -87,7 +89,11 @@ Cooldown requires publish timestamps in metadata. Registries without a "Yes" in brew install git-pkgs/git-pkgs/proxy ``` -Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases). +Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases), or run the container image: + +```bash +docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest +``` ### Helm @@ -104,6 +110,12 @@ using SQLite and filesystem artifact storage under `/data`. See [`deploy/charts/proxy/values.yaml`](deploy/charts/proxy/values.yaml) for ingress, external database and object-storage configuration options. +The SQLite database needs a local disk or a block-backed volume, even with one +replica; `ReadWriteOnce` alone doesn't guarantee that, since the storage class +can still be NFS. If the cluster only offers network filesystem storage, use +Postgres. Artifact storage can stay on NFS, separately from the database. See +[SQLite](docs/configuration.md#sqlite). + To trust a private CA, for example when upstreams sit behind a TLS-intercepting corporate proxy or are internal registries signed by an in-house CA, mount the certificate with `extraVolumes` and `extraVolumeMounts` and add its directory @@ -192,7 +204,7 @@ the proxy, which fetches them from the package's registry and caches them; attestations of versions held back by cooldown or blocked by the denylist are not served. -To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). +To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds). ### Cargo @@ -276,10 +288,10 @@ The equivalent environment variables are `PROXY_UPSTREAM_HOMEBREW_API` and `PROX ### Hex (Elixir) -Configure in `~/.hex/hex.config`: +Configure the mirror: -```erlang -{default_url, <<"http://localhost:8080/hex">>}. +```bash +mix hex.config mirror_url http://localhost:8080/hex ``` Or set the environment variable: @@ -288,6 +300,14 @@ Or set the environment variable: export HEX_MIRROR=http://localhost:8080/hex ``` +When the proxy applies a cooldown, also set: + +```bash +export HEX_UNSAFE_REGISTRY=1 +``` + +Filtering releases removes the registry signature. The proxy preserves the package name and repository fields, so origin verification can stay enabled. Without a cooldown, leave signature verification enabled too. + ### pub.dev (Dart/Flutter) Set the PUB_HOSTED_URL environment variable: @@ -347,17 +367,15 @@ Configure in `settings.gradle.kts`: ```kotlin buildCache { - local { - enabled = false - } remote { url = uri("http://localhost:8080/gradle/") - isPush = System.getenv("CI") != null } } ``` -The proxy rejects uploads unless `gradle.build_cache.read_only` is set to `false`, see [docs/configuration.md](docs/configuration.md#gradle-build-cache). +Enable build caching with `org.gradle.caching=true` in `gradle.properties`, or pass `--build-cache` when running Gradle. The local cache stays enabled and remote uploads are disabled by default. + +To upload from CI, set `gradle.build_cache.read_only` to `false` on the proxy and add `isPush = System.getenv("CI") != null` inside the client's `remote` block. See [docs/configuration.md](docs/configuration.md#gradle-build-cache). ### NuGet @@ -399,7 +417,16 @@ Or set globally: composer config -g repositories.proxy composer http://localhost:8080/composer ``` -To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). +`composer audit`, `audit.block-insecure` and Packagist's malware filter list +work through the proxy when `upstream.composer_repository` offers them in its +`packages.json`, as repo.packagist.org does. The proxy advertises them in its +own `packages.json` and passes the advisories and filter list requests through +to the upstream uncached. Names of packages in `upstream.composer_routes` or +`hosted.composer_vendors` are removed from advisories requests before they are +sent. Versions withheld by [cooldown](#version-cooldown) are not excluded from +the report. + +To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds). ### Conan (C/C++) @@ -442,10 +469,13 @@ default_channels: - http://localhost:8080/conda/main ``` +`default_channels` also routes requests for `defaults` through the proxy, including `-c defaults` and `environment.yml` files that name it. + Or set via command: ```bash conda config --add channels http://localhost:8080/conda/main +conda config --add default_channels http://localhost:8080/conda/main ``` pixi and other rattler-based clients use sharded repodata ([CEP 16](https://github.com/conda/ceps/blob/main/cep-0016.md)) where a channel offers it, as conda-forge does. Point them at the same channel URLs, for example `pixi init -c http://localhost:8080/conda/conda-forge`. The shard index and shards are cached as metadata when `cache_metadata` is on. @@ -468,6 +498,8 @@ local({ }) ``` +Source packages, Windows and macOS binaries (including the `bin/macosx/big-sur-arm64/` layout used by R 4.3 and later) are cached as artifacts. `PACKAGES` indexes and the `src/contrib/Meta/*.rds` files that `remotes::install_version` and pak read are cached as metadata. + ### Julia Set the Pkg server before starting Julia: @@ -695,16 +727,20 @@ indexes unchanged. To keep APT from selecting a denied version, configure ### RPM / Yum / DNF -Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`: +In the `[fedora]` section of your existing repository file under `/etc/yum.repos.d/`, replace the `metalink` or `mirrorlist` entry with: ```ini -[proxy-fedora] -name=Fedora via Proxy baseurl=http://localhost:8080/rpm/releases/$releasever/Everything/$basearch/os/ -enabled=1 -gpgcheck=0 ``` +In the `[updates]` section of `fedora-updates.repo`, replace its `metalink` or `mirrorlist` entry with: + +```ini +baseurl=http://localhost:8080/rpm/updates/$releasever/Everything/$basearch/ +``` + +Keep the existing `gpgkey` and signature settings, including `gpgcheck=1`. The proxy serves RPM packages unchanged, so their signatures remain valid. + Then: ```bash @@ -734,6 +770,11 @@ signature verification keeps working. Indexes use the metadata cache (`metadata_ttl`, stale fallback); `.apk` packages are stored in the shared artifact cache and remain available when the upstream is unreachable. +Set `upstream.apk_snapshots` (e.g. `"90d"`) to keep replaced indexes and serve +them by date at `/apk/alpine@2026-10-01/v3.22/main`, so pinned package versions +stay installable after the mirror moves on. See +[Alpine APK snapshots](docs/configuration.md#alpine-apk-snapshots). + To proxy other mirrors or private repositories, configure named upstreams under `upstream.apk` (this replaces the built-in default; re-add `alpine` if you still want it): @@ -1048,7 +1089,7 @@ Run with config file: ### PostgreSQL -SQLite is the default and works well for single-node deployments. For multi-node setups or if you prefer a managed database, switch to Postgres: +SQLite is the default and works well for single-node deployments, with the database file on a local disk or block-backed volume. For multi-node setups, when only network filesystem storage such as NFS is available, or if you prefer a managed database, switch to Postgres: ```yaml database: @@ -1170,6 +1211,9 @@ proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21 # Mirror a container image for one platform proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20 + +# Mirror a commit of an upstream.git repository into the git cache +proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 ``` The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror. @@ -1280,6 +1324,8 @@ Recently cached: | `GET /maven/*` | Maven repository protocol | | `GET /nuget/*` | NuGet V3 API | | `GET /composer/*` | Composer/Packagist protocol | +| `GET`, `POST /composer/api/security-advisories/` | Composer security advisories, passed through to upstream without private package names | +| `GET /composer/lists/summary.json` | Composer filter list summary, passed through to upstream uncached | | `GET /conan/*` | Conan C/C++ protocol | | `GET /chef/*` | Chef Supermarket protocol (universe, cookbook API and downloads) | | `GET /conda/*` | Conda/Anaconda protocol | @@ -1317,6 +1363,17 @@ Registered when `hosted.enabled` is set. Pushing and deleting require `Authoriza | `GET /api/hosted/{ecosystem}/{name}` | List the archives pushed for a package | | `DELETE /api/hosted/{ecosystem}/{name}/{version}` | Delete a pushed version | +### Projects API + +Registered when `projects.enabled` is set. Uploading and deleting require `Authorization: Bearer `. See [Project inventory](docs/configuration.md#project-inventory). + +| Endpoint | Description | +|----------|-------------| +| `PUT /api/projects/{project}/sbom` | Replace a project's recorded package versions with the ones the SBOM in the body lists | +| `GET /api/projects` | List projects, or with `?purl=pkg:npm/lodash@4.17.20` the projects using a package or version | +| `GET /api/projects/{project}` | A project's recorded package versions | +| `DELETE /api/projects/{project}` | Delete a project | + ### Cache API Registered when `cache_delete` is set. It has no authentication of its own. @@ -1478,9 +1535,7 @@ Response: { "cached_artifacts": 142, "total_size_bytes": 523456789, - "total_size": "499.2 MB", - "storage_url": "file:///path/to/cache/artifacts", - "database_path": "./cache/proxy.db" + "total_size": "499.2 MB" } ``` @@ -1520,6 +1575,7 @@ The proxy serves a web UI under `/ui`. No separate frontend build is needed -- t - **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts. - **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews. - **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files. +- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated. Package pages then show which projects use each version. ## Monitoring @@ -1538,6 +1594,7 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre | `proxy_metadata_cache_entries` | gauge | `ecosystem` | Number of cached metadata documents per ecosystem. | | `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) | | `proxy_metadata_evicted_total` | counter | `ecosystem` | Cached metadata documents evicted by the size limit | +| `proxy_metadata_expired_by_change_feed_total` | counter | `ecosystem` | Cached metadata documents set to revalidate because an upstream [change feed](docs/configuration.md#change-feeds) listed them | | `proxy_fetch_lease_waits_total` | counter | `kind` | Cache misses (`artifact` or `metadata`) that waited for another instance sharing the database to fetch them | | `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream | | `proxy_upstream_errors_total` | counter | `ecosystem`, `error_type` | Upstream fetch failures | @@ -1608,7 +1665,7 @@ The `kind` label on `proxy_cache_hits_total` and `proxy_cache_misses_total` keep `ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which. -**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. +**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_metadata_expired_by_change_feed_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. **From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all. @@ -1715,13 +1772,21 @@ sudo systemctl start proxy ### Docker -A Dockerfile is included in the repo. Build and run: +Images for linux/amd64 and linux/arm64 are published to `ghcr.io/git-pkgs/proxy` for each release, tagged with the version and `latest`: + +```bash +docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest +``` + +To build the image from the repo instead: ```bash docker build -t proxy . docker run -p 8080:8080 -v proxy-data:/data proxy ``` +The image has a health check that requests `/health` on port 8080. It checks the database and storage, so a container reports unhealthy when either fails. If you change the port with `PROXY_LISTEN`, override it with `--health-cmd`. + With `--read-only`, also pass `--tmpfs /tmp`: the proxy writes some uploads to a temporary file before reading them. With Postgres and S3: diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 258fba3..7c51732 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -8,7 +8,7 @@ // @securityDefinitions.apikey BearerAuth // @in header // @name Authorization -// @description "Bearer " followed by hosted.token +// @description "Bearer " followed by hosted.token, or projects.token for /api/projects // // The proxy caches package artifacts from upstream registries (npm, cargo, etc.) // providing faster, more reliable access for development teams. @@ -93,6 +93,9 @@ // PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB") // PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only // PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only +// PROXY_PROJECTS_ENABLED - Enable the project inventory (/api/projects, /ui/projects) +// PROXY_PROJECTS_TOKEN - Bearer token for uploading and deleting project SBOMs +// PROXY_PROJECTS_MAX_AGE - Remove projects not uploaded for this long (e.g. "90d") // PROXY_URL - Proxy URL for proxy hosted // // Example: @@ -492,6 +495,7 @@ func runMirror() { os.Exit(1) } cfg.LoadFromEnv() + printEnvWarnings(cfg) if *storageURL != "" { cfg.Storage.URL = *storageURL @@ -554,6 +558,7 @@ func runMirror() { proxy.CacheMetadata = true // mirror always caches metadata proxy.MetadataTTL = cfg.ParseMetadataTTL() proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() + proxy.MetadataStreamMaxSize = cfg.ParseMetadataStreamMaxSize() // As serve does, resolve Swift source repositories in the denylist through // the upstream, and refuse to run when one can't be. if err := server.ConfigureDenylist(cfg, proxy); err != nil { @@ -591,7 +596,9 @@ func runMirror() { return } + closeGit := useCommandGit(cfg, proxy, m, logger) progress, err := m.Run(ctx, source) + closeGit() if err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) @@ -630,6 +637,27 @@ func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler. return m } +// useCommandGit opens the git cache for the pkg:git items of proxy mirror, +// when upstream.git is configured, and returns the function that closes it. +// Where it can't be opened, such as while the running server holds +// git.cache_dir, those items fail with the reason and the rest are mirrored. +func useCommandGit(cfg *config.Config, proxy *handler.Proxy, m *mirror.Mirror, logger *slog.Logger) func() { + git, closeGit, err := server.NewMirrorGit(cfg, proxy, logger) + if err != nil { + m.Git = mirror.UnavailableGit{Reason: fmt.Sprintf("this command can't open the git cache (%v); while the server runs, use its mirror API", err)} + return func() {} + } + if git == nil { + return func() {} + } + m.Git = git + return func() { + if err := closeGit(); err != nil { + logger.Warn("closing the git cache", "error", err) + } + } +} + // mirrorExitCode returns the exit status for a finished mirror run: 1 when any // package version failed to mirror, 0 otherwise. func mirrorExitCode(progress *mirror.Progress) int { @@ -798,6 +826,14 @@ func formatSize(bytes int64) string { } } +// printEnvWarnings reports environment values LoadFromEnv ignored. serve +// logs them from server.Start; mirror never gets there. +func printEnvWarnings(cfg *config.Config) { + for _, warning := range cfg.EnvWarnings() { + fmt.Fprintf(os.Stderr, "warning: %s\n", warning) + } +} + func loadConfig(path string) (*config.Config, error) { if path != "" { return config.Load(path) diff --git a/cmd/proxy/main_test.go b/cmd/proxy/main_test.go index 3f17c27..d5a020d 100644 --- a/cmd/proxy/main_test.go +++ b/cmd/proxy/main_test.go @@ -57,3 +57,24 @@ func TestServeHelpProcess(*testing.T) { os.Args = []string{"proxy", "serve", "-help"} main() } + +func TestMirrorLogsEnvWarnings(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorEnvWarningsProcess$") + // The negative size stops mirror before it opens any storage. + cmd.Env = append(os.Environ(), + "PROXY_TEST_MIRROR_ENV_WARNINGS=1", + "PROXY_CACHE_METADATA=maybe", + "PROXY_STORAGE_MAX_SIZE=-1") + output, _ := cmd.CombinedOutput() + if !strings.Contains(string(output), "PROXY_CACHE_METADATA") { + t.Errorf("mirror did not warn about PROXY_CACHE_METADATA:\n%s", output) + } +} + +func TestMirrorEnvWarningsProcess(*testing.T) { + if os.Getenv("PROXY_TEST_MIRROR_ENV_WARNINGS") != "1" { + return + } + os.Args = []string{"proxy", "mirror", "pkg:npm/lodash@4.17.21"} + main() +} diff --git a/cmd/proxy/mirror_git_test.go b/cmd/proxy/mirror_git_test.go new file mode 100644 index 0000000..2789536 --- /dev/null +++ b/cmd/proxy/mirror_git_test.go @@ -0,0 +1,132 @@ +package main + +import ( + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +// gitMirrorUpstream serves team/repo, with one commit on main, from git +// http-backend, and returns the server and the commit. +func gitMirrorUpstream(t *testing.T) (*httptest.Server, string) { + t.Helper() + git := func(dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) + } + backend := "" + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if path := filepath.Join(git("", "--exec-path"), name); fileExistsForTest(path) { + backend = path + } + } + if backend == "" { + t.Skip("git http-backend is not installed") + } + root := t.TempDir() + bare := filepath.Join(root, "team", "repo.git") + git("", "init", "--quiet", "--bare", "--initial-branch=main", bare) + work := t.TempDir() + git("", "clone", "--quiet", bare, work) + git(work, "commit", "--quiet", "--allow-empty", "-m", "first") + git(work, "push", "--quiet", "origin", "HEAD:main") + head := git(work, "rev-parse", "HEAD") + + server := httptest.NewServer(&cgi.Handler{ + Path: backend, + Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}, + }) + t.Cleanup(server.Close) + return server, head +} + +func fileExistsForTest(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// writeGitMirrorConfig writes a config with upstream.git "up" at +// upstreamURL and the git cache, storage and database under dir. +func writeGitMirrorConfig(t *testing.T, dir, upstreamURL string) string { + t.Helper() + cfg := "storage:\n" + + " url: file://" + filepath.Join(dir, "artifacts") + "\n" + + "database:\n" + + " path: " + filepath.Join(dir, "proxy.db") + "\n" + + "upstream:\n" + + " allow_loopback: true\n" + + " git:\n" + + " up: " + upstreamURL + "\n" + + "git:\n" + + " cache_dir: " + filepath.Join(dir, "git") + "\n" + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil { + t.Fatalf("writing config: %v", err) + } + return path +} + +func TestMirrorCommandCachesGitCommit(t *testing.T) { + upstream, head := gitMirrorUpstream(t) + dir := t.TempDir() + configPath := writeGitMirrorConfig(t, dir, upstream.URL) + purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head + + output := runMirrorProcess(t, 0, "-config", configPath, purl) + if !strings.Contains(output, "1 downloaded") { + t.Errorf("output:\n%s", output) + } + cached, _ := filepath.Glob(filepath.Join(dir, "git", "up-*", "team", "repo.git")) + if len(cached) != 1 { + t.Errorf("cached repositories %v, want team/repo.git", cached) + } + + output = runMirrorProcess(t, 0, "-config", configPath, purl) + if !strings.Contains(output, "1 skipped") { + t.Errorf("second run output:\n%s", output) + } +} + +// TestMirrorCommandGitCacheInUse checks pkg:git items fail, pointing at the +// mirror API, while the running server holds git.cache_dir. +func TestMirrorCommandGitCacheInUse(t *testing.T) { + upstream, head := gitMirrorUpstream(t) + dir := t.TempDir() + configPath := writeGitMirrorConfig(t, dir, upstream.URL) + cache, err := gitcache.New(gitcache.Options{ + CacheDir: filepath.Join(dir, "git"), + RefreshWait: time.Second, + Client: &http.Client{}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = cache.Close() }() + purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head + + output := runMirrorProcess(t, 1, "-config", configPath, purl) + for _, want := range []string{"in use by another process", "mirror API", "1 failed"} { + if !strings.Contains(output, want) { + t.Errorf("output missing %q:\n%s", want, output) + } + } +} diff --git a/config.example.yaml b/config.example.yaml index d3b6da4..a99dbeb 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -116,12 +116,15 @@ storage: # "pkg:generic/owner%2Frepo": "90d" # How often to look for expired artifacts; at least "1m". sweep_interval: "10m" + # Keep this many of the newest cached versions of each package from + # eviction by age (cargo, golang, npm and pypi). 0 keeps none. + # keep_latest: 2 # Store fetched artifacts. Set to false to stream every download from # upstream without storing it; metadata filtering, cooldown and the # denylist still apply. Useful when another cache sits in front of the - # proxy. false is incompatible with direct_serve, scanning, mirror_api and - # mirror.schedules. + # proxy. false is incompatible with direct_serve, scanning, mirror_api, + # mirror.schedules and mirror.refresh. cache_artifacts: true # Redirect cached artifact downloads to presigned storage URLs (HTTP 302) @@ -133,12 +136,15 @@ storage: # How long presigned URLs remain valid (e.g. "5m", "1h"). Default: "15m". direct_serve_ttl: "15m" - # Public base URL to substitute into presigned URLs. Set this when the - # proxy reaches storage at an internal address (127.0.0.1, a Docker - # service name) but clients must use a public hostname. Only scheme and - # host are used; the signed path and query are preserved. For S3/MinIO - # the reverse proxy at this address must forward requests with the - # internal Host header or the SigV4 signature will not validate. + # Public base URL for presigned URLs. Set this when the proxy reaches + # storage at an internal address (127.0.0.1, a Docker service name, or + # VPC DNS) but clients must use a public hostname. Only scheme and host + # are used. For S3/MinIO the URL is signed for this host, so the server + # there must receive that Host header: a reverse proxy or CDN that sends + # the internal or bucket host to the origin fails signature validation. + # With virtual-hosted addressing (no use_path_style) the URL carries no + # bucket name, so this must be the bucket's own hostname. For GCS and + # Azure the host is replaced after signing. # direct_serve_base_url: "https://minio.example.com" # Database configuration @@ -189,6 +195,10 @@ upstream: # npm_routes: # "@example/*": "https://npm.example.com/registry" + # Follow the npm registry's change feed, so a new release is fetched on the + # next request instead of after metadata_ttl. Requires cache_metadata. + # npm_change_feed: true + # Cargo sparse index URL cargo: "https://index.crates.io" @@ -245,6 +255,11 @@ upstream: # composer_routes: # "example/*": "https://composer.example.com/packages" + # Follow the change feed of the Packagist at composer, so a new release is + # fetched on the next request instead of after metadata_ttl. Requires + # cache_metadata. + # composer_change_feed: true + # Conan registry URL conan: "https://center.conan.io" @@ -442,6 +457,15 @@ gradle: # npm_scopes: ["@acme"] # composer_vendors: ["acme"] +# Project inventory (disabled by default). Repositories upload an SBOM to +# PUT /api/projects/{project}/sbom, and the UI shows which project uses +# which package versions. Project names are visible on the unauthenticated UI. +# projects: +# enabled: true +# token: "${PROXY_PROJECTS_TOKEN}" +# # Drop projects that have not uploaded for this long ("" keeps them) +# max_age: "90d" + # Re-mirror container image tags on a schedule, so a moved tag is picked up # even when nobody pulls it. An unchanged tag costs one HEAD request. # mirror: @@ -449,6 +473,16 @@ gradle: # - images: ["pkg:docker/library/alpine@3.20", "pkg:docker/library/nginx@stable"] # interval: "6h" # platforms: ["linux/amd64"] +# +# Mirror new releases of the most used packages in the background, so the +# first install after a release doesn't wait on the upstream. Cooldown and +# the denylist apply. +# refresh: +# interval: "24h" +# top: 50 # packages per ecosystem, by recent downloads +# since: "14d" +# max_versions: 200 # per run; default no cap +# max_bytes: "2GB" # per run; default no cap # Health endpoint configuration. health: @@ -464,6 +498,13 @@ health: # to spot malicious releases before they're pulled into projects. # Supported durations: "7d" (days), "48h" (hours), "30m" (minutes), "0" (disabled) cooldown: + # Independent policies selected through /cooldown/{name}/{registry}/. + # profiles: + # 3d: + # default: "3d" + # 7d: + # default: "7d" + # Global default cooldown for all ecosystems # default: "3d" diff --git a/deploy/charts/proxy/values.yaml b/deploy/charts/proxy/values.yaml index 5936ef3..bbb253f 100644 --- a/deploy/charts/proxy/values.yaml +++ b/deploy/charts/proxy/values.yaml @@ -79,6 +79,12 @@ extraInitContainers: [] # image: busybox:1.37 # command: ["sh", "-c", "until nc -z postgres 5432; do sleep 2; done"] +# With SQLite, the database file needs a local disk or block-backed volume, +# even with one replica. ReadWriteOnce doesn't guarantee that: pick a +# storageClass that isn't NFS or another network filesystem. If only network +# filesystem storage is available, set config.data.database to Postgres. +# Artifact storage can stay on NFS on a separate volume (extraVolumes and +# extraVolumeMounts, with config.data.storage.url pointing at it). persistence: enabled: true # Keep this aligned with config.data.storage.url and config.data.database.path diff --git a/docs/architecture.md b/docs/architecture.md index 24af2fb..1d17426 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -304,6 +304,11 @@ HTTP protocol handlers for each registry type. - `handleIndex()` - Proxy sparse index - `handleDownload()` - Serve cached crate +**DebianHandler:** +- Serves `/debian` and `/debian/{name}` mirrors of APT archives (`upstream.debian_repositories`) +- Routes `.deb` packages to the artifact cache, `dists/…` to the metadata cache, and proxies all other paths +- Keys the artifact cache on name, version, and filename, scoped by repository for named archives + **SwiftHandler:** - Proxies the Swift Package Registry v1 read endpoints - Rewrites release URLs and caches source archives @@ -316,7 +321,7 @@ HTTP protocol handlers for each registry type. ### `internal/metadata/npm` -The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. It imports no database, storage, metrics, HTTP or logging packages, which a test checks. +The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. `ScanPackument` and `RewriteStream` do the same rewrite over a reader in two passes, the first collecting the time map, dist-tags and version names and the second copying each kept version, so the handler can rewrite a packument from storage without holding it in memory. It imports no database, storage, metrics, HTTP or logging packages, which a test checks. ### `internal/jsonscan` @@ -363,10 +368,14 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g ### `internal/mirror` -Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. +Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and the commit or tag, with its submodules, in the git cache that `/git` serves. The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. +### `internal/inventory` + +Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated marker on the project pages. + ### `internal/config` Configuration loading. diff --git a/docs/configuration.md b/docs/configuration.md index 3e94531..1ac89a6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,6 +2,8 @@ The proxy can be configured via command line flags, environment variables, or a configuration file. Command line flags take precedence over environment variables, which take precedence over the configuration file. +Boolean environment variables accept `true`, `t`, `1`, `yes` or `on`, and `false`, `f`, `0`, `no` or `off`, in any case. Any other value, such as a typo, is ignored: the setting keeps its value from the configuration file or default, and the proxy logs a warning at startup. + ## Configuration File Create a YAML or JSON file and pass it with `-config`: @@ -65,10 +67,11 @@ storage: |--------|-------------|------|-------------| | `storage.url` | `PROXY_STORAGE_URL` | `-storage-url` | Storage URL (file:// or s3://) | | `storage.path` | `PROXY_STORAGE_PATH` | `-storage-path` | Local path (deprecated, use url) | -| `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB") | +| `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB"). Negative or out-of-range sizes are rejected at startup | | `storage.cache_artifacts` | `PROXY_STORAGE_CACHE_ARTIFACTS` | - | Store fetched artifacts (default: true); `false` streams them from upstream | | `storage.retention.default` | `PROXY_STORAGE_RETENTION_DEFAULT` | - | Evict artifacts not served for this long (e.g. "30d"); see [Retention](#retention) | | `storage.retention.sweep_interval` | `PROXY_STORAGE_RETENTION_SWEEP_INTERVAL` | - | How often expired artifacts are looked for (default: "10m", at least "1m") | +| `storage.retention.keep_latest` | - | - | Keep this many of the newest cached versions of each package from eviction by age (default: 0); see [Retention](#retention) | `storage.max_size` counts cached artifacts and cached metadata documents (`cache_metadata`). When the total goes over the limit, the least recently served entries of either kind are evicted first; an evicted metadata document is fetched from upstream again on the next request for it. Pushed artifacts are not counted or evicted. An artifact replaced by a refetch stays in storage for at least an hour, or `storage.direct_serve_ttl` if longer, so requests already reading it can finish, and storage use can exceed the limit by what was replaced in that time. @@ -91,7 +94,7 @@ storage: Every download is a fresh upstream fetch, and concurrent requests for the same artifact are not combined. Artifacts with a digest known up front (OCI blobs, Swift archives, Helm charts) are verified while streaming: the response is sent chunked, and on a mismatch the connection is aborted before the response completes so the client never receives a tampered artifact as a good one. The same happens when the upstream connection fails mid-download. -`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve`, `mirror_api` or `mirror.schedules`, which all need stored artifacts, and the `mirror` command refuses to run with it. +`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve`, `mirror_api`, `mirror.schedules` or `mirror.refresh`, which all need stored artifacts, and the `mirror` command refuses to run with it. ### Retention @@ -111,10 +114,21 @@ storage: sweep_interval: "10m" ``` -Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`. +Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. The blobs of an image that `mirror.schedules` keeps current also count from the last run that found them cached, so the schedule keeps them; for `max_size` eviction such a run doesn't count as use. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`. A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version, written in one of the forms the table below lists for the ecosystem. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems, except `generic`, which requires an explicit rule. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention supports (`covered`) and which ones `default` does not reach (`default_not_applied`), including `generic`. +`keep_latest` keeps the newest versions of each package, however long ago they were served: + +```yaml +storage: + retention: + default: "30d" + keep_latest: 2 +``` + +With this, the files of the two highest cached versions of a package are never evicted by age, and older versions expire by their rule as before. Versions are ordered by the ecosystem's version rules, not by when they were fetched, and a prerelease counts like any other version. Only versions in the cache count, not those upstream, so the kept ones need not be the latest releases. `keep_latest` applies to `cargo`, `golang`, `npm` and `pypi`; the other ecosystems ignore it, since many of them cache files under a digest or a build rather than a version. On startup the proxy logs the ecosystems it reaches (`keep_latest_covered`). `storage.max_size` can still evict a kept version. + Generic retention is opt-in because release assets may disappear upstream and cannot then be downloaded again after eviction. An existing `default` never starts evicting generic downloads. Set `ecosystems.generic: "30d"` to cover all generic artifacts, or configure only selected GitHub repositories: ```yaml @@ -201,6 +215,8 @@ database: The database runs in WAL mode. Writes go through a single connection, since SQLite allows one writer at a time. Reads use a separate pool of read-only connections (one per CPU, between 2 and 8), so request lookups don't wait behind writes, stats queries or eviction scans. An in-memory database (`:memory:`) uses one connection for both. +The SQLite database needs a local disk or a block-backed volume, even with one replica. SQLite's WAL mode doesn't work on network filesystems such as NFS ([sqlite.org/wal.html](https://www.sqlite.org/wal.html)), which shows up as stalled requests and `database disk image is malformed`. A Kubernetes `ReadWriteOnce` volume doesn't guarantee suitable storage, since its storage class can still be NFS. If only network filesystem storage is available to the proxy, use `driver: postgres`. Artifact storage can stay on NFS, on a separate volume from the SQLite database. + ### PostgreSQL ```yaml @@ -295,6 +311,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.allow_loopback` | `PROXY_UPSTREAM_ALLOW_LOOPBACK` | `false` | | `upstream.npm` | `PROXY_UPSTREAM_NPM` | `https://registry.npmjs.org` | | `upstream.npm_full_metadata` | `PROXY_UPSTREAM_NPM_FULL_METADATA` | `false` | +| `upstream.npm_change_feed` | `PROXY_UPSTREAM_NPM_CHANGE_FEED` | `false` (see [change feeds](#change-feeds)) | | `upstream.cargo` | `PROXY_UPSTREAM_CARGO` | `https://index.crates.io` | | `upstream.cargo_download` | `PROXY_UPSTREAM_CARGO_DOWNLOAD` | `https://static.crates.io/crates` | | `upstream.gem` | `PROXY_UPSTREAM_GEM` | `https://rubygems.org` | @@ -312,6 +329,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.nuget_search` | `PROXY_UPSTREAM_NUGET_SEARCH` | `https://azuresearch-usnc.nuget.org` | | `upstream.composer` | `PROXY_UPSTREAM_COMPOSER` | `https://packagist.org` | | `upstream.composer_repository` | `PROXY_UPSTREAM_COMPOSER_REPOSITORY` | `https://repo.packagist.org` | +| `upstream.composer_change_feed` | `PROXY_UPSTREAM_COMPOSER_CHANGE_FEED` | `false` (see [change feeds](#change-feeds)) | | `upstream.conan` | `PROXY_UPSTREAM_CONAN` | `https://center.conan.io` | | `upstream.chef` | `PROXY_UPSTREAM_CHEF` | `https://supermarket.chef.io` | | `upstream.conda` | `PROXY_UPSTREAM_CONDA` | `https://conda.anaconda.org` | @@ -323,6 +341,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.rpm` | `PROXY_UPSTREAM_RPM` | `https://dl.fedoraproject.org/pub/fedora/linux` | | `upstream.homebrew_api` | `PROXY_UPSTREAM_HOMEBREW_API` | `https://formulae.brew.sh/api` | | `upstream.homebrew_artifact` | `PROXY_UPSTREAM_HOMEBREW_ARTIFACT` | `https://ghcr.io` | +| `upstream.apk_snapshots` | `PROXY_UPSTREAM_APK_SNAPSHOTS` | disabled; see [Alpine APK snapshots](#alpine-apk-snapshots) | Private, ULA, CGNAT, and loopback addresses are rejected by default. Add each private upstream hostname or IP address to `upstream.allow_private_hosts`. The matching environment variable accepts a comma-separated list. Loopback upstreams also require `upstream.allow_loopback: true`. That setting permits upstream requests and redirects to reach any loopback address. @@ -348,6 +367,8 @@ upstream: composer_repository: "https://upstream-proxy.example.com/composer" ``` +The proxy reads `packages.json` from `upstream.composer_repository` and keeps it in memory for the metadata TTL, or an hour when the TTL is 0, whether or not `cache_metadata` is on. A failed fetch is not retried within that time: the last good copy is used, or the entries are left out. It passes on its `security-advisories` and `filter` entries (Packagist's malware list) with URLs pointing at the proxy. An upstream that offers neither is still usable; `composer audit` then reports nothing from it. A filter `api-url` is not passed on; Composer reads filter entries from package metadata instead. + `upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint. `upstream.chef` is the Supermarket served at `/chef/`, for Berkshelf (`source "https://proxy.example.com/chef"`) and Policyfiles (`default_source :supermarket, "https://proxy.example.com/chef"`). The upstream has to serve the Supermarket API: `/universe`, `/api/v1/cookbooks/{name}`, its versions and their downloads. A private Supermarket that needs credentials takes them from `upstream.auth`, scoped to its URL. A download redirect to another host, such as the storage bucket behind Supermarket, does not receive them: @@ -388,6 +409,7 @@ upstream: - Composer metadata of routed packages carries an empty `notification-url`, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. - Use `upstream.auth` for the registry's credentials. The proxy has no authentication of its own, so every client that can reach it can download routed packages. - Search (`/composer/search.json`) and the package list (`/composer/packages/list.json`) still come from the default upstream only. +- Composer security advisories and filter lists come from the default upstream. Routed package names, and those in `hosted.composer_vendors`, are removed from advisories requests before they are sent, so they only get advisories that their route's own metadata carries. - Attestations of a routed npm version (`/npm/-/npm/v1/attestations/...`) come from the route, like its tarballs. - npm audit, signing-key, search and ping requests still go to `upstream.npm`. An audit request lists every package in the dependency tree, routed ones included. - Routes can only be set in the configuration file, not with environment variables. @@ -576,6 +598,8 @@ with the metadata TTL and served byte-for-byte unchanged so apk signature verification keeps working; `.apk` packages use the shared artifact cache. When `upstream.apk` is empty, a single repository named `alpine` pointing at the official mirror is available; configuring any entry replaces that default. +Earlier index versions can be kept and served by date; see +[Alpine APK snapshots](#alpine-apk-snapshots). ```yaml upstream: @@ -623,6 +647,53 @@ as signatures and keys, is passed through uncached. When `upstream.freebsd` is e repository named `freebsd` pointing at `https://pkg.freebsd.org` is available; configuring any entry replaces that default. +### Alpine APK snapshots + +```yaml +upstream: + # Keep replaced APK indexes for 90 days. + apk_snapshots: "90d" +``` + +Alpine mirrors only keep the current build of each package, so after an +upgrade `apk add curl=8.5.0-r0` fails because the index no longer lists that +build. With `upstream.apk_snapshots` set, the proxy keeps every version of an +index (and of a detached signature) it stores, and serves them under a dated +repository name: + +``` +http://localhost:8080/apk/alpine@2026-10-01/v3.22/main +``` + +`/apk/{name}@{YYYY-MM-DD}/…` returns the index that was current at the end of +that day (UTC). The kept files are the signed upstream bytes, so apk verifies +them as usual. A day before the first stored version of an index, or a day +that has not started yet, is `404`. Today's snapshot can still change until the +day ends. Packages under a snapshot path are the same as under the plain path: +they come from the artifact cache, or from upstream if not cached; a build the +upstream has dropped and the proxy never cached is `404`. A configured +repository whose name contains `@` takes precedence over a snapshot path. + +Snapshots are recorded when an index is requested through the plain path, so +they only exist for repositories clients use. A version counts from the first +time the proxy served it, which can be up to `metadata_ttl` after upstream +published it. A version is deleted once it was replaced more than the +configured duration ago; the newest version of each index is always kept. +Durations take a `d` suffix; empty or `"0"` disables snapshots. Snapshots need +`cache_metadata`. Their files live under `_snapshots/` in storage and do not +count towards `storage.max_size`. + +Turning snapshots off, or `cache_metadata` off, leaves the kept versions in +place. To remove them, first set a short duration such as `"1h"` and let the +hourly prune delete everything but the newest version of each index. + +Snapshots keep indexes, not packages. A package listed by an old snapshot can +still be evicted by `storage.max_size` or [retention](#retention), and is then +only available while upstream still has it. Retention counts from a package's +last download, so a pinned build nobody installed for longer than the `alpine` +retention is evicted even while a snapshot lists it. Use package rules with +`"0"` for builds that must stay installable. + ### Go checksum database ```yaml @@ -842,8 +913,12 @@ commits, the request waits for their fetch, so a clone right after a push gets them; if the fetch fails or stalls, the cached copy is served. A `stale_max_age` of `0` never serves a copy the upstream has not just confirmed. A repository the upstream reports missing (404), refuses (401, 403) -or redirects is not served from the cache. Redirects are never followed, so -point the upstream at a repository's current location. +or redirects is not served from the cache, also to requests that skip the ref +listing and go straight to `git-upload-pack`, until a later check succeeds. +Redirects are never followed, so point the upstream at a repository's current +location. A `git-upload-pack` request checks the upstream only when the last +check refused the repository or the copy was last checked more than a minute +past `stale_max_age` ago, so a clone checks once, at its ref listing. The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it). The `git` processes never reach the network themselves: their fetches go @@ -889,6 +964,12 @@ those yourself. The proxy locks `cache_dir` while it runs, through a `.gitcache-lock` file in it, so a second process given the same directory, such as another replica on a shared volume, fails to start; give each its own. +Each repository's directory is named by its path as requested. Where +`cache_dir` ignores case in file names, as on macOS and Windows by default, +paths that differ only in case (`acme/widgets` and `acme/WIDGETS`) share one +directory and are treated as one repository. Use a case-sensitive file system +for an upstream whose repositories differ only in case. + Git LFS objects are cached in the artifact cache, so they use the configured storage backend and retention rather than `cache_dir`. git-lfs finds the LFS API under the rewritten remote, at `/git/{name}/{repo}.git/info/lfs`, with no @@ -947,6 +1028,13 @@ one doesn't fall back to the Gradle Plugin Portal. Files are cached per repository, so the same coordinates in two repositories never share a cache entry. +In every repository, a file the proxy would cache (an artifact, checksum or +signature) whose group directories contain a dot or colon +(`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, is not in +the Maven layout and gets 404 without an upstream request. Its group would read +the same as `org/foo`'s. Other paths there, such as `maven-metadata.xml` or a +directory listing, are handled as anywhere else. + `upstream.maven_repositories` is additive. `/maven/…` keeps serving `upstream.maven` with unchanged cache identities. A repository name is matched before the default repository, so it hides every group ID whose first segment @@ -964,6 +1052,18 @@ A repository's files are cached under its name and a hash of its upstream URL, so pointing a repository at a different URL stops serving what the old one sent. +In every Maven repository, `maven-metadata.xml` follows `metadata_ttl`, and +its `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from the +cached metadata rather than fetched, so they always match it. Other files are +cached for good, except SNAPSHOT files named with `SNAPSHOT` rather than a +build timestamp (`lib-1.0-SNAPSHOT.jar`, as JitPack and non-unique snapshot +repositories publish them), which each build replaces: those are fetched +again for every `GET`, through the denylist, cooldown, scanning and the +Plugin Portal fallback like any other file. The new copy replaces the cached +one once stored; while upstream can't be reached the cached copy is served, +and `HEAD` is answered from cache. Timestamped SNAPSHOT files +(`lib-1.0-20240101.120000-1.jar`) never change and are cached. + This field has no environment variable, as with the other named upstream maps. `PROXY_UPSTREAM_MAVEN` still sets `upstream.maven`. @@ -1091,6 +1191,37 @@ Uploads are off by default, so a client that can reach the proxy cannot overwrit `max_age` and `max_size` are independent and can be combined. When both are set, age-based eviction runs first, then size-based eviction trims remaining entries oldest-first. +## Cooldown profiles + +Set `cooldown.profiles` to offer several policies from one instance. Clients select a profile through their package manager's registry URL: + +```yaml +cooldown: + default: "0" + profiles: + 3d: + default: "3d" + 7d: + default: "7d" + packages: + "pkg:npm/@example/tool": "0" +``` + +For the `3d` profile, use these paths after your proxy's base URL: + +| Client | Path | +|--------|------| +| npm registry | `/cooldown/3d/npm/` | +| Go `GOPROXY` | `/cooldown/3d/go` | +| Cargo registry | `sparse+https://your-proxy.example/cooldown/3d/cargo/` | +| Bundler RubyGems mirror | `/cooldown/3d/gem` | + +The prefix works with all package protocol routes. Cooldown has an effect only for [registries with cooldown support](../README.md#supported-registries). Each profile defines its own `default`, `ecosystems`, `packages` and `package_patterns`; it does not inherit the global cooldown overrides. Existing unprefixed URLs, the UI, APIs and background mirror jobs use the global policy. + +Profile names contain lowercase letters, digits, hyphens and underscores, and start with a letter or digit. Profiles must be configured before startup; an unknown profile returns 404. Nested profiles are rejected. Profile paths expose package protocols only, so `/cooldown/3d/ui/` and `/cooldown/3d/api/` return 404. + +Database records, artifacts and upstream metadata are shared. Generated download URLs retain the profile prefix, and filtered metadata and its validators are separated by profile. Because npm packuments are shared too, cooldown in the global policy or in any profile makes every URL fetch and serve full packuments, as `upstream.npm_full_metadata` does. Changing a registry URL does not rewrite artifact URLs already recorded in a lockfile; regenerate those URLs if the selected policy must cover the whole installation. + ## Cooldown The cooldown feature hides package versions published too recently, giving the community time to spot malicious releases before they reach your projects. When a version is within its cooldown period, it's stripped from metadata responses so package managers won't install it. @@ -1127,13 +1258,13 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. -Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` or Gradle `.module` itself is downloaded, which clients do before a version's other files, and otherwise with a `HEAD` request. A `.pom` or `.module` refused this way is still cached, so it counts toward `max_size` before the version leaves cooldown. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. +Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` or Gradle `.module` itself is downloaded, which clients do before a version's other files, and otherwise with a `HEAD` request. A `.pom` or `.module` refused this way is still cached, so it counts toward `max_size` before the version leaves cooldown. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served, as they are without cooldown. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. Maven caveats: @@ -1144,15 +1275,21 @@ Maven caveats: Helm uses each release's `created` time from the repository's `index.yaml`. Releases in cooldown are left out of the index the proxy serves, and their archives aren't served; a release without `created` is let through. Charts pulled over OCI have no publish time and aren't held back. +Cargo, RubyGems, Hex, pub.dev, Conda and Helm check artifact downloads against cooldown, including files cached before the policy changed. When a publish-time lookup fails, an already-cached file is still served; an uncached download fails with 502, or 503 with `Retry-After` when the rate-limit gate holds the request. Conda checks timestamps per file because builds of one version can have different publish times. As in conda, a repodata `timestamp` is read as milliseconds when it is above 253402300799 (the end of year 9999 in seconds) and as seconds otherwise, since some older entries use seconds. + +Conda repodata and Helm indexes are parsed once per cached document for download checks. With `cache_metadata` off, the parsed index is kept for one minute, including the fetch, so an install can reuse it across files. Conda streams repodata when its compressed copy exceeds `metadata_max_size`, under the same decompressed size bound as the metadata filter. + RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, and the filtered result is kept until the first version it held back leaves cooldown (see [rewritten metadata cache](#rewritten-metadata-cache)); a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. -With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead, and `npm dist-tag ls` shows the same tags. +With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead; other tags that name a version in cooldown are removed, as for the denylist, and `npm dist-tag ls` shows the same tags. A tarball request for a version the cached packument doesn't list yet fetches the packument again, within `metadata_ttl` or not, and is decided on what the upstream lists now; if the packument can't be fetched, the download is not let through, as when a publish time can't be read. + +In pub.dev package metadata, when `latest` is in cooldown it names the newest stable version out of cooldown instead, or the newest prerelease when no stable version is left, and it is left out when every version is in cooldown. Conda cooldown covers `repodata.json`, `current_repodata.json`, `repodata_from_packages.json` and sharded repodata (CEP 16). Other forms of the repodata, such as `repodata.json.zst`, `repodata.json.bz2` and `repodata.jlap` patches, would list versions still in cooldown, so with cooldown on they are answered 404 and conda and mamba fall back to `repodata.json`. With `cache_metadata`, `repodata.json` is read through the metadata cache as it is without cooldown: within `metadata_ttl` the stored copy is filtered as it is read from storage, after it the upstream is asked with its ETag, and while the upstream is down the stale copy is filtered. Once a document has been filtered, the proxy remembers the ETag of the result until the first entry it left out leaves cooldown, so a client revalidating with `If-None-Match` gets `304 Not Modified` without the document being filtered or sent again. The first response for each document has no ETag, as it is sent while it is filtered. Without `cache_metadata`, `repodata.json` is fetched and filtered on every request. Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. -Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. +Hex cooldown requires `HEX_UNSAFE_REGISTRY=1` because filtering releases re-encodes the protobuf payload without the original signature. The proxy preserves the package name and repository fields, so origin verification can stay enabled. `HEX_NO_VERIFY_REPO_ORIGIN` controls origin verification and does not disable signature verification. Without a cooldown, leave signature verification enabled too. ## Version denylist @@ -1176,6 +1313,13 @@ Names and versions are compared the way each ecosystem treats them: - PyPI names ignore case, and runs of `-`, `_` and `.` are equivalent (PEP 503), so `pkg:pypi/zope.interface@6.0` also covers `zope_interface-6.0-*.whl`. - NuGet ids and versions ignore case. +- Debian, RPM, Alpine, Arch and FreeBSD entries ignore the distribution + namespace: `pkg:deb/debian/curl@8.5.0-2` and `pkg:deb/curl@8.5.0-2` are the + same entry. +- Debian and RPM filenames leave out the epoch, so an entry with one also + covers the version without it: `pkg:deb/foo@1:2.3-1` denies + `foo_2.3-1_amd64.deb`. An entry without an epoch only matches a version + without one. - A RubyGems entry also covers that version's platform gems: `pkg:gem/nokogiri@1.16.0` denies `nokogiri-1.16.0-x86_64-linux.gem`. - Composer versions ignore a leading `v`, which Packagist keeps from the tag: @@ -1204,7 +1348,7 @@ Names and versions are compared the way each ecosystem treats them: channel only. Denied versions are removed from npm packuments, Cargo sparse indexes, and PyPI -simple HTML/JSON and release metadata, including responses served from cached +simple HTML/JSON (files and the JSON `versions` list) and release metadata, including responses served from cached metadata during an upstream outage. No publication timestamp or enabled cooldown is required. npm tags pointing at denied versions are removed; `latest` is moved to the highest remaining stable version when available. Other tags are not @@ -1306,7 +1450,7 @@ scanning: | `scanning.scanners[].name` | - | Identifies this scanner in logs and metrics | | `scanning.scanners[].url` | - | Endpoint the proxy POSTs scan notifications to | | `scanning.scanners[].mode` | - | `block` (default) or `monitor` | -| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems | +| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems. Valid names: `alpine`, `arch`, `cargo`, `chef`, `composer`, `conan`, `conda`, `cran`, `deb`, `freebsd`, `gem`, `generic`, `git` (Git LFS objects), `golang`, `helm`, `hex`, `julia`, `maven`, `npm`, `nuget`, `oci`, `pub`, `pypi`, `rpm`, `swift`. The aliases `go`, `docker`, `rubygems`, `packagist`, `debian` and `apk` are accepted too; any other name stops the proxy at startup | | `scanning.scanners[].headers` | - | Extra HTTP headers sent with every scan request (e.g. for authenticating to the scanner service). Values support `${VAR_NAME}` expansion | ### How caching defers to a scan verdict @@ -1387,7 +1531,7 @@ cache_metadata: true Or via environment variable: `PROXY_CACHE_METADATA=true`. -Some metadata never changes once published, and is cached like an artifact, with or without `cache_metadata`, and served without revalidation whatever `metadata_ttl` says: a Go module's `.mod` and `.info` for a canonical version (a release, pre-release, pseudo-version or `+incompatible` version; the checksum database fixes their content), and RPM repodata files named after their own checksum (`repodata/-primary.xml.gz`). A stored copy is checked against the digest recorded for it and fetched again if it doesn't match. Go branch and commit queries, `@v/list` and `@latest`, and RPM `repomd.xml` and plainly named repodata files follow `metadata_ttl` as before. A checksum-named repodata file over `metadata_max_size` is streamed from upstream uncached. +Some metadata never changes once published, and is cached like an artifact, with or without `cache_metadata`, and served without revalidation whatever `metadata_ttl` says: a Go module's `.mod` and `.info` for a canonical version (a release, pre-release, pseudo-version or `+incompatible` version; the checksum database fixes their content), and RPM repodata files named after their own checksum (`repodata/-primary.xml.gz`). A stored copy is checked against the digest recorded for it and fetched again if it doesn't match. Go branch and commit queries, `@v/list` and `@latest`, and RPM `repomd.xml` follow `metadata_ttl` as before; plainly named repodata files follow the cached `repomd.xml` (see [Metadata TTL](#metadata-ttl)). A checksum-named repodata file over `metadata_max_size` is streamed from upstream uncached. Composer archive downloads look up their upstream URL in the same cached metadata, so an archive is resolved without asking Packagist again and stays installable while it is down. Only a version or file the cached metadata does not list yet is looked up upstream. With or without `cache_metadata`, a cached archive of a tagged release from the default repository is served without any metadata lookup; with cooldown enabled it is checked against the publish time recorded when the archive was first resolved. Branch (dev) versions, routed packages and archives picked by a query string such as `?sha=` are always resolved from the metadata. @@ -1407,17 +1551,40 @@ Or via environment variable: `PROXY_METADATA_TTL=10m`. Set to `"0"` to always revalidate with upstream (ETag-based conditional requests still avoid re-downloading unchanged content). -When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. +When upstream is unreachable and the cached entry is past its TTL (with a `metadata_ttl` of `"0"`, on every such request), the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. The `ETag` and `Last-Modified` sent with a document are always those of the bytes served, so a document fetched from upstream but not stored, for example because storage failed, carries the upstream's validators rather than the older cached copy's. + +APT and RPM clients fetch a release file first (`InRelease` or `Release`, `repomd.xml`) and reject an index file whose checksum doesn't match it. So an index file the cached release file lists (`Packages`, `Sources`, a plainly named `primary.xml.gz`, …) does not follow its own `metadata_ttl`: its cached copy is served while it matches the cached release file's sha256, however old, and otherwise it is fetched from upstream. If upstream's copy doesn't match either, the repository has changed since the release file was cached; that copy is served and the release file is expired, so the next `apt update` or `dnf makecache` gets a matching pair. Where a suite has both `InRelease` and `Release` cached, the one fetched last decides. The release file itself follows `metadata_ttl`. Index files it doesn't list, such as `by-hash` paths, follow `metadata_ttl` too. + +### Change feeds + +Within `metadata_ttl` a new release isn't seen. For npm and Composer the proxy can instead follow the registry's change feed, which lists the packages that changed, and revalidate just those: + +```yaml +cache_metadata: true +upstream: + npm_change_feed: true + composer_change_feed: true +``` + +Or via environment variables: `PROXY_UPSTREAM_NPM_CHANGE_FEED=true`, `PROXY_UPSTREAM_COMPOSER_CHANGE_FEED=true`. Both require `cache_metadata`, and do nothing with a `metadata_ttl` of `"0"`. + +Every minute the proxy reads what changed since its last read. Cached metadata of a listed package that was fetched before the read is marked as needing revalidation, and the next request for it asks the upstream with its `ETag` (a `304` when nothing changed). Artifacts are not touched. Listed packages the proxy hasn't cached cost nothing beyond the read. The feed is read in full, so the upstream doesn't learn which packages the proxy has cached. Packages in `upstream.npm_routes` or `upstream.composer_routes` are cached under other keys and never matched. + +- Composer reads `{upstream.composer}/metadata/changes.json`, which packagist.org advertises as `metadata-changes-url`. Both `vendor/package` and its `vendor/package~dev` file are covered. Packagist keeps 24 hours of changes; if the proxy's position is older than that, Packagist asks for a resync and all cached Composer metadata is revalidated. repo.packagist.org sends `Cache-Control: max-age=900`, but Packagist purges its CDN when a package changes, so revalidation gets the new file within seconds; the proxy doesn't apply that header either way. +- npm reads `https://replicate.npmjs.com/registry/_changes`, the public registry's feed, whatever `upstream.npm` is. It is only useful when `upstream.npm` is registry.npmjs.org or serves the same packages. Packuments (`/npm/{name}`) are covered; version and dist-tag documents (`/npm/{name}/latest`) still wait out `metadata_ttl`. A busy minute on npm lists a few thousand packages; the proxy reads up to 100,000 per pass and continues where it stopped on the next. + +A new release therefore shows up about a minute after the registry lists it rather than up to `metadata_ttl` later. The feed position is kept in memory: after a restart the proxy starts from the feed's current position, and changes made while it was down are covered by `metadata_ttl` as before. While a feed can't be read the proxy logs a warning and retries with a growing delay, up to 15 minutes, and `metadata_ttl` applies meanwhile. A request whose upstream fetch overlaps the change and the feed read can still store the old document for one TTL. Each instance sharing a database reads the feeds itself. `proxy_metadata_expired_by_change_feed_total` counts the cached documents marked, by ecosystem, and debug logging names each one. ### Metadata size limit -Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. +Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. With `cache_metadata` and the metadata rewrite cache on, an npm packument is streamed from the upstream into storage and rewritten from there in two passes, so it isn't held in memory and the limit doesn't apply to it; the rewritten copy is kept gzip-compressed. Requests for a single version (`//`) and tarball downloads read the version from the stored copy the same way. `metadata_stream_max_size` (1 GB by default) caps a document streamed this way; one over it isn't stored, and the stored copy, if any, is served instead. ```yaml -metadata_max_size: "100MB" # default +metadata_max_size: "100MB" # default +metadata_stream_max_size: "1GB" # default ``` -Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`. +Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`, and `PROXY_METADATA_STREAM_MAX_SIZE=2GB` for `metadata_stream_max_size`. ### Artifact size limit @@ -1530,6 +1697,7 @@ Pushed versions are listed in the package metadata clients resolve against, with - Pushed versions are added next to the upstream ones and are never held back by cooldown. npm `dist-tags` stay as the upstream has them, also in `npm dist-tag ls`. - A package the upstream doesn't know is served from its pushed versions alone. Its npm `latest` tag is the highest pushed version. - Pushed Composer dev versions (`dev-*`, `*-dev`) are listed in the `~dev` metadata file. +- Pushed Composer versions carry an empty `notification-url`, as routed packages do, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. A pushed `composer.json` that sets its own `notification-url` keeps it, and Composer then reports installs there. - npm entries carry the archive's sha512 as `dist.integrity`. Composer entries carry its sha256 as `dist.reference` and leave `shasum` empty. Metadata of packages with pushed versions is rewritten on every request instead of being cached, so a push or delete shows up on the next request. @@ -1569,6 +1737,89 @@ Generic files never come from an upstream: anything not pushed answers 404. Whil Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones. +## Project inventory + +The project inventory answers "which repository uses which version of a package?". Each repository uploads an SBOM of its default branch, and the proxy records the package versions it lists. The UI then shows: + +- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version. +- On a package page: which projects use each version, including versions the proxy hasn't cached. + +It is disabled by default. Uploads and deletes require a token: + +```yaml +projects: + enabled: true + token: "${PROXY_PROJECTS_TOKEN}" + max_age: "90d" +``` + +| Config | Environment | Description | +|--------|-------------|-------------| +| `projects.enabled` | `PROXY_PROJECTS_ENABLED` | Register the `/api/projects` endpoints and the UI pages (default `false`) | +| `projects.token` | `PROXY_PROJECTS_TOKEN` | Bearer token required to upload and delete; required when enabled, supports `${VAR}` | +| `projects.max_age` | `PROXY_PROJECTS_MAX_AGE` | Remove projects that haven't uploaded for this long, e.g. `90d` or `720h`; empty or `0` keeps them (default) | + +Removing old projects runs hourly. With several instances on one database, one of them does it. + +### Uploading an SBOM + +`PUT /api/projects/{project}/sbom` takes a CycloneDX (JSON or XML) or SPDX (JSON or tag-value) document of up to 32 MiB and replaces everything recorded for the project with what it lists. The optional `ref` and `branch` query parameters are stored and shown with the project. + +The project name is a path such as `group/sub/repo`: segments of letters, digits, `.`, `_` and `-`, at most 255 characters. It can be sent as path segments (`/api/projects/group/sub/repo/sbom`) or URL-encoded as one segment (`/api/projects/group%2Fsub%2Frepo/sbom`). + +```bash +syft dir:. -o cyclonedx-json > sbom.json +curl -X PUT --data-binary @sbom.json \ + -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ + "https://proxy.example.com/api/projects/acme/shop/web/sbom?ref=$(git rev-parse HEAD)&branch=main" +``` + +The response counts the recorded package versions and the skipped entries: + +```json +{"project":"acme/shop/web","components":9,"skipped":6} +``` + +Entries are recorded by their package URL, with the package named the way the proxy caches it: NuGet IDs lowercased, PyPI names normalized as in PEP 503 (`zope.interface` becomes `zope-interface`), the distribution dropped from `deb`, `rpm` and `apk` package URLs. Entries without a package URL (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`), entries without a version, and conda entries without a `channel` qualifier (the proxy records conda packages as `channel/name`) are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once. + +A GitLab CI job on the default branch could look like this: + +```yaml +sbom: + image: alpine:3.20 + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + script: + - apk add --no-cache curl + - curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin + - syft dir:. -o cyclonedx-json > sbom.json + - > + curl -sSf -X PUT --data-binary @sbom.json + -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" + "https://proxy.example.com/api/projects/$CI_PROJECT_PATH/sbom?ref=$CI_COMMIT_SHA&branch=$CI_COMMIT_BRANCH" +``` + +### Querying + +```bash +curl https://proxy.example.com/api/projects # all projects +curl https://proxy.example.com/api/projects/acme/shop/web # one project's package versions +curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash' # projects using lodash +curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash@4.17.20' # projects using lodash 4.17.20 +curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ + https://proxy.example.com/api/projects/acme/shop/web +``` + +Reading needs no token, like the rest of `/api`. With `?purl=`, each project carries the `version` it uses and no `component_count`. A delete names the project itself: `DELETE /api/projects/acme/shop/web/sbom` is answered with 400 unless a project `acme/shop/web/sbom` exists. + +A project's page in the UI also has a delete button. It asks for the token and sends it to `DELETE /ui/api/projects/{project}`, which works like the API endpoint and also requires the token; the token is kept for the browser session. Like `/ui/api/cache`, it sits under `/ui`, so rules for the UI at an ingress cover it. + +### What it shows + +The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. Versions are compared as the SBOM gives them: a version spelled differently from how the proxy caches it, such as Composer's `v6.4.1` for `6.4.1` or a Debian version with or without its epoch, shows as "not cached" on the package page. + +Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy. + ## Mirror API The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP: @@ -1624,7 +1875,7 @@ curl -X DELETE http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1 curl -X DELETE 'http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre?filename=guava-32.1.3-jre.jar' ``` -The name is the package name the UI shows, for example `group:artifact` for Maven. The response lists the deleted files. The next request for a deleted file fetches it from the upstream again. The stored bytes are removed after the same delay as a replaced artifact, so downloads already in progress can finish. +The name is the package name the UI shows, for example `group:artifact` for Maven. An RPM package's `filename` is its path under `upstream.rpm` (`updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm`), since repositories under one upstream can ship different files under one name. The response lists the deleted files. The next request for a deleted file fetches it from the upstream again. The stored bytes are removed after the same delay as a replaced artifact, so downloads already in progress can finish. Pushed archives are not touched: a whole-version delete skips them, and naming one answers 409. Delete those through the [hosted API](#hosted-packages). @@ -1648,6 +1899,8 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. +An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | @@ -1710,6 +1963,22 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403 - Dependencies aren't mirrored with a formula; list them too. brew also needs the JSON API, which the proxy keeps with `cache_metadata`. - Casks, source archives and taps other than homebrew/core aren't mirrored. +### Git repositories + +`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID or a tag: + +```bash +proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 +proxy mirror pkg:git/github.com/git-pkgs/purl@v0.2.1 +proxy mirror pkg:git/github.com/git-pkgs/purl +``` + +- The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. +- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. +- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. When the upstream can't be reached, a cached tag is mirrored at the commit it last pointed to, with a warning in the log. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID. +- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`. +- The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. + ### Scheduled refresh Tags such as `alpine:3.20` move. The server can re-mirror them on its own, so the cache follows the tag even when nobody pulls it or the proxy is offline later: @@ -1737,6 +2006,37 @@ Each run checks the tag with a HEAD request. If the tag still points to the cach - Mirrored layers are scanned when `scanning.enabled` is set. A blocked layer fails the run and keeps the previous image. - Schedules are read at startup; changes need a restart. They do not need `mirror_api`. +### Refreshing the most used packages + +For packages, the cache learns about a new release only when a client asks for it, and that first install waits on the upstream or fails while it is down. `mirror.refresh` mirrors new releases of the packages clients download most, so they are cached before anyone asks: + +```yaml +mirror: + refresh: + interval: 24h + top: 50 + since: 14d + max_versions: 200 + max_bytes: 2GB +``` + +| Setting | Env | Description | +|---------|-----|-------------| +| `mirror.refresh.interval` | - | How often the refresh runs, at least `1m` | +| `mirror.refresh.top` | - | Packages per ecosystem to refresh, the ones with the most downloads among those downloaded within `since`. Default: `50` | +| `mirror.refresh.since` | - | How recently a package must have been downloaded, with a `d` suffix for days. Default: `14d` | +| `mirror.refresh.max_versions` | - | Stop a run after mirroring this many new versions. Default: no cap | +| `mirror.refresh.max_bytes` | - | Stop a run once it has downloaded this much, such as `2GB`. It is checked before each version, so the last one can go over it. Default: no cap | + +For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Versions numbered by year, such as certifi's `2025.1.31`, are all one line. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. + +- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. +- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again from upstream, which doesn't count as a client fetching them, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`; without it the proxy logs a warning at startup and refreshes the other ecosystems only. +- Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. +- Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. A file a run mirrored counts as used only once a client downloads it, so a package nobody fetches drops out after `since`. +- Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. +- What a run mirrors is evicted by `storage.max_size` and retention like any other file. With `max_versions` and `max_bytes` a run stays bounded when many packages release at once. + ## Docker ### SQLite with Local Storage diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index e35a86f..fe8723d 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -460,6 +460,194 @@ const docTemplate = `{ } } }, + "/api/projects": { + "get": { + "description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "List projects", + "parameters": [ + { + "type": "string", + "description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20", + "name": "purl", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}": { + "get": { + "description": "Returns the project with the package versions recorded for it.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Get a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.", + "tags": [ + "projects" + ], + "summary": "Delete a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}/sbom": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.", + "consumes": [ + "application/json", + "application/xml", + "text/plain" + ], + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Upload a project's SBOM", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Commit the SBOM was generated from", + "name": "ref", + "in": "query" + }, + { + "type": "string", + "description": "Branch the SBOM was generated from", + "name": "branch", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectUploadResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/search": { "get": { "produces": [ @@ -1062,6 +1250,10 @@ const docTemplate = `{ "server.PackageResponse": { "type": "object", "properties": { + "cooldown": { + "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").", + "type": "string" + }, "description": { "type": "string" }, @@ -1120,6 +1312,106 @@ const docTemplate = `{ } } }, + "server.ProjectComponent": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + }, + "purl": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "server.ProjectListResponse": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectSummary" + } + } + } + }, + "server.ProjectResponse": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", + "type": "integer" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectComponent" + } + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectSummary": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", + "type": "integer" + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectUploadResponse": { + "type": "object", + "properties": { + "components": { + "description": "Components is the number of package versions recorded.", + "type": "integer" + }, + "project": { + "type": "string" + }, + "skipped": { + "description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.", + "type": "integer" + } + } + }, "server.SearchPackageResult": { "type": "object", "properties": { @@ -1169,9 +1461,6 @@ const docTemplate = `{ "cached_artifacts": { "type": "integer" }, - "database_path": { - "type": "string" - }, "downloaded": { "type": "string" }, @@ -1192,9 +1481,6 @@ const docTemplate = `{ "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", "type": "boolean" }, - "storage_url": { - "type": "string" - }, "total_size": { "type": "string" }, @@ -1206,7 +1492,7 @@ const docTemplate = `{ }, "securityDefinitions": { "BearerAuth": { - "description": "\"Bearer \" followed by hosted.token", + "description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects", "type": "apiKey", "name": "Authorization", "in": "header" diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 09e4f11..496aecd 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -453,6 +453,194 @@ } } }, + "/api/projects": { + "get": { + "description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "List projects", + "parameters": [ + { + "type": "string", + "description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20", + "name": "purl", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}": { + "get": { + "description": "Returns the project with the package versions recorded for it.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Get a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.", + "tags": [ + "projects" + ], + "summary": "Delete a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}/sbom": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.", + "consumes": [ + "application/json", + "application/xml", + "text/plain" + ], + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Upload a project's SBOM", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Commit the SBOM was generated from", + "name": "ref", + "in": "query" + }, + { + "type": "string", + "description": "Branch the SBOM was generated from", + "name": "branch", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectUploadResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/search": { "get": { "produces": [ @@ -1055,6 +1243,10 @@ "server.PackageResponse": { "type": "object", "properties": { + "cooldown": { + "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").", + "type": "string" + }, "description": { "type": "string" }, @@ -1113,6 +1305,106 @@ } } }, + "server.ProjectComponent": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + }, + "purl": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "server.ProjectListResponse": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectSummary" + } + } + } + }, + "server.ProjectResponse": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", + "type": "integer" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectComponent" + } + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectSummary": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", + "type": "integer" + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectUploadResponse": { + "type": "object", + "properties": { + "components": { + "description": "Components is the number of package versions recorded.", + "type": "integer" + }, + "project": { + "type": "string" + }, + "skipped": { + "description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.", + "type": "integer" + } + } + }, "server.SearchPackageResult": { "type": "object", "properties": { @@ -1162,9 +1454,6 @@ "cached_artifacts": { "type": "integer" }, - "database_path": { - "type": "string" - }, "downloaded": { "type": "string" }, @@ -1185,9 +1474,6 @@ "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", "type": "boolean" }, - "storage_url": { - "type": "string" - }, "total_size": { "type": "string" }, @@ -1199,7 +1485,7 @@ }, "securityDefinitions": { "BearerAuth": { - "description": "\"Bearer \" followed by hosted.token", + "description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects", "type": "apiKey", "name": "Authorization", "in": "header" diff --git a/go.mod b/go.mod index db6695b..f1cb4f7 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,27 @@ module github.com/git-pkgs/proxy -go 1.26.7 - -toolchain go1.27.3 +go 1.27.3 require ( github.com/BurntSushi/toml v1.6.0 github.com/Masterminds/semver/v3 v3.5.0 - github.com/aws/aws-sdk-go-v2/config v1.33.9 - github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 + github.com/aws/aws-sdk-go-v2 v1.47.5 + github.com/aws/aws-sdk-go-v2/config v1.33.12 + github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 + github.com/aws/smithy-go v1.28.7 github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 - github.com/git-pkgs/cooldown v0.3.0 + github.com/git-pkgs/cooldown v0.3.1 github.com/git-pkgs/enrichment v0.7.5 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/registries v0.13.0 - github.com/git-pkgs/sbom v0.1.7 + github.com/git-pkgs/sbom v0.1.9 github.com/git-pkgs/spdx v0.3.3 + github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 github.com/klauspost/compress v1.20.3 @@ -28,6 +30,7 @@ require ( github.com/prometheus/client_golang v1.25.0 github.com/prometheus/client_model v0.6.3 github.com/swaggo/swag v1.16.6 + github.com/ulikunitz/xz v0.5.17 github.com/vmihailenco/msgpack/v5 v5.4.1 gocloud.dev v0.47.0 golang.org/x/mod v0.42.0 @@ -51,24 +54,21 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.47.3 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect - github.com/aws/smithy-go v1.28.5 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -76,7 +76,6 @@ require ( github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/pom v0.1.8 // indirect - github.com/git-pkgs/vers v0.7.2 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.4 // indirect @@ -108,7 +107,6 @@ require ( github.com/rogpeppe/go-internal v1.16.0 // indirect github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect - github.com/ulikunitz/xz v0.5.17 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.46.0 // indirect diff --git a/go.sum b/go.sum index d1c0563..cb9226b 100644 --- a/go.sum +++ b/go.sum @@ -52,46 +52,46 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= -github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= -github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= -github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= -github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= -github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= +github.com/aws/aws-sdk-go-v2/config v1.33.12 h1:IV1YARaXahwCqUIyP8IlWsP85Lil+5ozTozDQXPumRk= +github.com/aws/aws-sdk-go-v2/config v1.33.12/go.mod h1:h51R06bzMVwNBRZOX5ylHhNDnkB/azZeq/LvxM6q2pQ= +github.com/aws/aws-sdk-go-v2/credentials v1.20.12 h1:6P7Q7yH/GIwHg6MLKv3ySsBqaCoE1z+C0OTzYMenhEY= +github.com/aws/aws-sdk-go-v2/credentials v1.20.12/go.mod h1:Kw/SaoW/wc7IqJErqXJIzKo8b7Aelxfbuc6XlbrRnsc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 h1:vsFkmFrfzODkxVbyuPLscv5j5lC8KbG48LZwV+p56Y0= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3/go.mod h1:pjQYVtvVBEtlsf2p6nXMsjPuDaWs9GM44FMdHbmnmYU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 h1:163OpmhcdzTzgbXbLV+u2v1LY/QmGT148/eOwkczKCY= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5/go.mod h1:uDza1Eipo/Olii3CCgLglHmAni24PScSAhpfIKbUl0c= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= +github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1 h1:EJlz0wSlZDKaCN2FHwg9RBR+C+gsuVdYblJnEnrq3M4= +github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1/go.mod h1:6rVJFO9ZO+4j/QL5/FbtAsbb8xHC85j79wXA9ndOCTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 h1:CO485zrLOZBeq8mrdsudc8cjyhvC4LY4JFt8ZfHvXZ0= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2/go.mod h1:cBAjVUKBr+5wnx1K3XWS8u5AZsqpA2S/0z7YtI6BGFc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 h1:Enx5JcIy/o90kio1LFZ/HguxIsos3clbGuSpGdyshX8= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2/go.mod h1:Ldmll8/2VqaVerWWDf8IGG+6pAvZY04gs3tevq3nIFs= github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 h1:4PokON/ISKuD6Zl062n4WfQNnEFBMHvfeJy73Jp7K+g= github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0/go.mod h1:6ZFrmv/qIAYFmG+2WhA7mFU8+edldRlISwT+tNnyLuA= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM= -github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= -github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 h1:3vu/eUqh0rjh7Zy4N4YKcpZX4Aud+zinB3VsLhhlQFw= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.0/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 h1:kIEpD5dUry7VET3Ybl7zj7PPHRB4lb52G9iTz+a1Tqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.0/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 h1:WfFev+HgTodggcNon5HieiLxxwWwb8Br8R2+nDkixvk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= @@ -121,8 +121,8 @@ github.com/git-pkgs/archives v0.10.0 h1:/qgk0MI6VK2sKOoVTd0B4zs5hwTo+St0X8cYYftz github.com/git-pkgs/archives v0.10.0/go.mod h1:GJ/h7NPTUYiPinOUxpYNsHFCUikYZcJXvugBvP1VGNc= github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx2xk= github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= -github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= -github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= +github.com/git-pkgs/cooldown v0.3.1 h1:XDt54+ZuYoJclTLysaQ2GUZPhYYuJdjpaqeLSwjMdto= +github.com/git-pkgs/cooldown v0.3.1/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= github.com/git-pkgs/enrichment v0.7.5 h1:jbfFzf7+HSS7el8AuMri4vHXq2izIIZDVIU0uYI5LFM= github.com/git-pkgs/enrichment v0.7.5/go.mod h1:TbxC/nGQN96YCbdoiDs8a9z3gk2zXmLVS1989/jvnsU= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= @@ -137,8 +137,8 @@ github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= -github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= -github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= +github.com/git-pkgs/sbom v0.1.9 h1:sfF5cJAwmgrm2N4HTXQWjiBe8tdu8xwhjwfAOyDNMfc= +github.com/git-pkgs/sbom v0.1.9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= diff --git a/internal/config/config.go b/internal/config/config.go index c09a1ba..f93c996 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -64,12 +64,14 @@ import ( "encoding/base64" "encoding/json" "fmt" + "math" "net" "net/url" "os" "path" "path/filepath" "regexp" + "slices" "sort" "strconv" "strings" @@ -170,6 +172,11 @@ type Config struct { // size return ErrMetadataTooLarge. Default: "100MB". MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"` + // MetadataStreamMaxSize is the largest metadata document the proxy + // streams from upstream into storage instead of buffering it, which + // metadata_max_size doesn't cover (e.g. "1GB"). Default: "1GB". + MetadataStreamMaxSize string `json:"metadata_stream_max_size" yaml:"metadata_stream_max_size"` + // ArtifactMaxSize is the largest upstream artifact the proxy will fetch // (e.g. "2GB"). Larger artifacts are refused and not cached. Default: "" // (unlimited). "0" is also unlimited. @@ -213,6 +220,9 @@ type Config struct { // Hosted configures pushing packages to the proxy (/api/hosted). Hosted HostedConfig `json:"hosted" yaml:"hosted"` + // Projects configures the project inventory (/api/projects). + Projects ProjectsConfig `json:"projects" yaml:"projects"` + // Gradle configures Gradle HttpBuildCache behavior. Gradle GradleConfig `json:"gradle" yaml:"gradle"` @@ -229,6 +239,9 @@ type Config struct { // CooldownConfig configures version cooldown periods. // Versions published more recently than the cooldown are hidden from metadata responses. type CooldownConfig struct { + // Profiles are independent policies exposed at /cooldown/{name}/. + Profiles map[string]CooldownConfig `json:"profiles,omitempty" yaml:"profiles,omitempty"` + // Default is the global default cooldown (e.g., "3d", "48h", "0" to disable). Default string `json:"default" yaml:"default"` @@ -425,9 +438,55 @@ func (s *ScannerConfig) Validate() error { default: return fmt.Errorf("invalid mode %q (must be block or monitor)", s.Mode) } + for i, eco := range s.Ecosystems { + key, err := ScannerEcosystemKey(eco) + if err != nil { + return err + } + s.Ecosystems[i] = key + } return nil } +// scannerEcosystems are the ecosystem keys handlers pass to scanners: the +// retention keys plus the handlers retention does not cover, and git for +// Git LFS objects. +var scannerEcosystems = func() []string { + keys := append(slices.Clone(retention.KnownKeys), "arch", "chef", "git") + slices.Sort(keys) + return keys +}() + +// scannerEcosystemAliases maps common ecosystem names to handler keys. +var scannerEcosystemAliases = map[string]string{ + "apk": "alpine", + "debian": "deb", + "docker": "oci", + "go": "golang", + "packagist": "composer", + "rubygems": "gem", +} + +// NormalizeScannerEcosystem returns the handler key for a +// scanning.scanners[].ecosystems entry, resolving case and common aliases. +func NormalizeScannerEcosystem(eco string) string { + eco = strings.ToLower(strings.TrimSpace(eco)) + if key, ok := scannerEcosystemAliases[eco]; ok { + return key + } + return eco +} + +// ScannerEcosystemKey is NormalizeScannerEcosystem that also rejects +// names no handler uses. +func ScannerEcosystemKey(eco string) (string, error) { + key := NormalizeScannerEcosystem(eco) + if !slices.Contains(scannerEcosystems, key) { + return "", fmt.Errorf("unknown ecosystem %q (valid: %s)", eco, strings.Join(scannerEcosystems, ", ")) + } + return key, nil +} + // StorageConfig configures artifact storage. type StorageConfig struct { // URL is the storage backend URL. @@ -460,10 +519,10 @@ type StorageConfig struct { // Uses Go duration syntax (e.g. "5m", "1h"). Default: "15m". DirectServeTTL string `json:"direct_serve_ttl" yaml:"direct_serve_ttl"` - // DirectServeBaseURL overrides the scheme and host of presigned URLs - // before returning them to clients. Useful when the proxy reaches - // storage at an internal address (e.g. 127.0.0.1 or a Docker hostname) - // but clients must use a public one. + // DirectServeBaseURL sets the scheme and host of presigned URLs. Useful + // when the proxy reaches storage at an internal address (e.g. 127.0.0.1 + // or a Docker hostname) but clients must use a public one. S3 URLs are + // signed for this host; GCS and Azure URLs have it replaced after signing. DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"` // CacheArtifacts stores fetched artifacts so later downloads are served @@ -500,6 +559,10 @@ type RetentionConfig struct { // SweepInterval is how often expired artifacts are looked for, at least // one minute. Default: "10m". SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"` + + // KeepLatest is how many of the highest cached versions of each package + // are never evicted by age, in the ecosystems that support it. Default: 0. + KeepLatest int `json:"keep_latest" yaml:"keep_latest"` } // GradleConfig configures Gradle-specific features. @@ -614,6 +677,12 @@ type UpstreamConfig struct { // wins. NPMRoutes map[string]string `json:"npm_routes" yaml:"npm_routes"` + // NPMChangeFeed follows the public npm registry's change feed and + // revalidates cached packuments of packages listed in it on their next + // request, rather than serving them until metadata_ttl runs out. + // Requires cache_metadata. Default: false. + NPMChangeFeed bool `json:"npm_change_feed" yaml:"npm_change_feed"` + // Cargo is the upstream cargo index URL. // Default: https://index.crates.io Cargo string `json:"cargo" yaml:"cargo"` @@ -711,6 +780,12 @@ type UpstreamConfig struct { // matching pattern wins. ComposerRoutes map[string]string `json:"composer_routes" yaml:"composer_routes"` + // ComposerChangeFeed follows the metadata change feed of + // upstream.composer and revalidates cached metadata of packages listed + // in it on their next request, rather than serving it until + // metadata_ttl runs out. Requires cache_metadata. Default: false. + ComposerChangeFeed bool `json:"composer_change_feed" yaml:"composer_change_feed"` + // Conan is the upstream Conan registry URL. // Default: https://center.conan.io Conan string `json:"conan" yaml:"conan"` @@ -773,6 +848,12 @@ type UpstreamConfig struct { // Default when empty: {"alpine": "https://dl-cdn.alpinelinux.org/alpine"}. APK map[string]string `json:"apk" yaml:"apk"` + // APKSnapshots keeps earlier versions of Alpine APK indexes for this + // long after upstream replaced them, and serves them at + // /apk/{name}@{YYYY-MM-DD}/. Accepts a "d" suffix, e.g. "90d". + // Empty or "0" disables snapshots. + APKSnapshots string `json:"apk_snapshots" yaml:"apk_snapshots"` + // Arch maps repository names to Arch Linux pacman mirror base URLs, // served at /arch/{name}/. The remaining request path mirrors the // upstream layout, e.g. /arch/archlinux/core/os/x86_64/core.db. @@ -881,6 +962,11 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.apk", u.APK); err != nil { return err } + if u.APKSnapshots != "" { + if _, err := parseRetentionDuration("upstream.apk_snapshots", u.APKSnapshots); err != nil { + return err + } + } if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil { return err } @@ -998,6 +1084,12 @@ func (c *Config) validateUpstream() error { if c.Upstream.GoSumDBsOffline && !c.CacheMetadata { return fmt.Errorf("upstream.go_sumdbs_offline requires cache_metadata") } + if c.Upstream.NPMChangeFeed && !c.CacheMetadata { + return fmt.Errorf("upstream.npm_change_feed requires cache_metadata") + } + if c.Upstream.ComposerChangeFeed && !c.CacheMetadata { + return fmt.Errorf("upstream.composer_change_feed requires cache_metadata") + } return nil } @@ -1350,11 +1442,33 @@ func setEnvString(dst *string, key string) { } } -// setEnvBool is setEnvString for boolean fields, parsed via envBool. -func setEnvBool(dst *bool, key string) { - if v := os.Getenv(key); v != "" { - *dst = envBool(v) +// parseEnvBool accepts what strconv.ParseBool does plus yes/no/on/off, +// ignoring case and surrounding space. +func parseEnvBool(v string) (bool, bool) { + v = strings.ToLower(strings.TrimSpace(v)) + switch v { + case "yes", "on": + return true, true + case "no", "off": + return false, true } + b, err := strconv.ParseBool(v) + return b, err == nil +} + +// setEnvBool is setEnvString for boolean fields, parsed via parseEnvBool. +// Any other value leaves *dst unchanged and is reported via EnvWarnings. +func (c *Config) setEnvBool(dst *bool, key string) { + v := os.Getenv(key) + if v == "" { + return + } + b, ok := parseEnvBool(v) + if !ok { + c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping %t", key, v, *dst)) + return + } + *dst = b } // setGradleReadOnlyFromEnv parses strictly: an unrecognised value keeps @@ -1365,8 +1479,8 @@ func (c *Config) setGradleReadOnlyFromEnv() { if v == "" { return } - readOnly, err := strconv.ParseBool(strings.TrimSpace(v)) - if err != nil { + readOnly, ok := parseEnvBool(v) + if !ok { c.Gradle.BuildCache.ReadOnly = true c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping the Gradle build cache read-only", key, v)) return @@ -1414,16 +1528,16 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Listen, "PROXY_LISTEN") setEnvString(&c.BaseURL, "PROXY_BASE_URL") setEnvString(&c.UIBaseURL, "PROXY_UI_URL") - setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR") - setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES") - setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS") + c.setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR") + c.setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES") + c.setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS") setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL") setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH") setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE") - setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") + c.setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL") setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL") - setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS") + c.setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS") setEnvString(&c.Storage.Retention.Default, "PROXY_STORAGE_RETENTION_DEFAULT") setEnvString(&c.Storage.Retention.SweepInterval, "PROXY_STORAGE_RETENTION_SWEEP_INTERVAL") setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER") @@ -1434,10 +1548,11 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT") setEnvString(&c.AccessLog.Path, "PROXY_ACCESS_LOG_PATH") setEnvStringSlice(&c.Upstream.AllowPrivateHosts, "PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS") - setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK") + c.setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK") setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM") - setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA") - setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE") + c.setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA") + c.setEnvBool(&c.Upstream.NPMChangeFeed, "PROXY_UPSTREAM_NPM_CHANGE_FEED") + c.setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE") setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO") setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD") setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM") @@ -1454,6 +1569,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.NuGetSearch, "PROXY_UPSTREAM_NUGET_SEARCH") setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER") setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY") + c.setEnvBool(&c.Upstream.ComposerChangeFeed, "PROXY_UPSTREAM_COMPOSER_CHANGE_FEED") setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN") setEnvString(&c.Upstream.Chef, "PROXY_UPSTREAM_CHEF") setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA") @@ -1462,25 +1578,30 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.Swift, "PROXY_UPSTREAM_SWIFT") setEnvString(&c.Upstream.OCIDefault, "PROXY_UPSTREAM_OCI_DEFAULT") setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN") + setEnvString(&c.Upstream.APKSnapshots, "PROXY_UPSTREAM_APK_SNAPSHOTS") setEnvString(&c.Upstream.RPM, "PROXY_UPSTREAM_RPM") setEnvString(&c.Upstream.HomebrewAPI, "PROXY_UPSTREAM_HOMEBREW_API") setEnvString(&c.Upstream.HomebrewArtifact, "PROXY_UPSTREAM_HOMEBREW_ARTIFACT") setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT") - setEnvBool(&c.Scanning.Enabled, "PROXY_SCANNING_ENABLED") - setEnvBool(&c.Scanning.FailOpen, "PROXY_SCANNING_FAIL_OPEN") + c.setEnvBool(&c.Scanning.Enabled, "PROXY_SCANNING_ENABLED") + c.setEnvBool(&c.Scanning.FailOpen, "PROXY_SCANNING_FAIL_OPEN") setEnvString(&c.Scanning.Timeout, "PROXY_SCANNING_TIMEOUT") setEnvString(&c.Scanning.SigningKey, "PROXY_SCANNING_SIGNING_KEY") setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL") - setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") - setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") - setEnvBool(&c.CacheDelete, "PROXY_CACHE_DELETE") - setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") + c.setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") + c.setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") + c.setEnvBool(&c.CacheDelete, "PROXY_CACHE_DELETE") + c.setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN") setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES") setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS") + c.setEnvBool(&c.Projects.Enabled, "PROXY_PROJECTS_ENABLED") + setEnvString(&c.Projects.Token, "PROXY_PROJECTS_TOKEN") + setEnvString(&c.Projects.MaxAge, "PROXY_PROJECTS_MAX_AGE") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") + setEnvString(&c.MetadataStreamMaxSize, "PROXY_METADATA_STREAM_MAX_SIZE") setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE") setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT") @@ -1563,8 +1684,8 @@ func (c *Config) Validate() error { // Validate direct serve TTL if specified if c.Storage.DirectServeTTL != "" { - if _, err := time.ParseDuration(c.Storage.DirectServeTTL); err != nil { - return fmt.Errorf("invalid storage.direct_serve_ttl %q: %w", c.Storage.DirectServeTTL, err) + if _, err := parseNonNegativeDuration("storage.direct_serve_ttl", c.Storage.DirectServeTTL); err != nil { + return err } } @@ -1581,8 +1702,8 @@ func (c *Config) Validate() error { // Validate metadata TTL if specified if c.MetadataTTL != "" && c.MetadataTTL != "0" { - if _, err := time.ParseDuration(c.MetadataTTL); err != nil { - return fmt.Errorf("invalid metadata_ttl %q: %w", c.MetadataTTL, err) + if _, err := parseNonNegativeDuration("metadata_ttl", c.MetadataTTL); err != nil { + return err } } @@ -1614,6 +1735,8 @@ func (c *Config) validateCacheArtifacts() error { return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served") case len(c.Mirror.Schedules) > 0: return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror.schedules: mirrored artifacts would never be served") + case c.Mirror.Refresh != nil: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror.refresh: mirrored artifacts would never be served") case c.Hosted.Enabled: return fmt.Errorf("storage.cache_artifacts: false cannot be combined with hosted.enabled: pushed packages would never be served") } @@ -1621,6 +1744,10 @@ func (c *Config) validateCacheArtifacts() error { } func (c *Config) validateComponents() error { + if err := c.Cooldown.ValidateProfiles(); err != nil { + return err + } + if err := validateMetadataRewriteCacheSize(c.MetadataRewriteCacheSize); err != nil { return err } @@ -1653,6 +1780,10 @@ func (c *Config) validateComponents() error { HostedGenericRepository, HostedGenericRepository) } + if err := c.Projects.Validate(); err != nil { + return err + } + if err := c.Mirror.Validate(); err != nil { return err } @@ -1727,6 +1858,7 @@ const ( defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default defaultHitFlushInterval = time.Second defaultMetadataMaxSize = 100 << 20 + defaultMetadataStreamMaxSize = 1 << 30 defaultMetadataRewriteCacheSize = 256 << 20 defaultGradleBuildCacheMaxUploadSize = 100 << 20 defaultGradleBuildCacheSweepInterval = 10 * time.Minute @@ -1780,22 +1912,25 @@ func validateRequestTimeout(field, s string) error { return nil } -func validateMetadataMaxSize(s string) error { +func validateMetadataMaxSize(name, s string) error { if s == "" { return nil } size, err := ParseSize(s) if err != nil { - return fmt.Errorf("invalid metadata_max_size: %w", err) + return fmt.Errorf("invalid %s: %w", name, err) } if size <= 0 { - return fmt.Errorf("invalid metadata_max_size %q: must be positive", s) + return fmt.Errorf("invalid %s %q: must be positive", name, s) } return nil } func (c *Config) validateDownloadSizeLimits() error { - if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil { + if err := validateMetadataMaxSize("metadata_max_size", c.MetadataMaxSize); err != nil { + return err + } + if err := validateMetadataMaxSize("metadata_stream_max_size", c.MetadataStreamMaxSize); err != nil { return err } return validateArtifactMaxSize(c.ArtifactMaxSize) @@ -1858,6 +1993,19 @@ func (c *Config) ParseMetadataMaxSize() int64 { return size } +// ParseMetadataStreamMaxSize returns the size limit for metadata streamed +// into storage in bytes. Returns 1GB if unset or invalid. +func (c *Config) ParseMetadataStreamMaxSize() int64 { + if c.MetadataStreamMaxSize == "" { + return defaultMetadataStreamMaxSize + } + size, err := ParseSize(c.MetadataStreamMaxSize) + if err != nil || size <= 0 { + return defaultMetadataStreamMaxSize + } + return size +} + // ParseArtifactMaxSize returns the artifact size limit in bytes, or 0 for // no limit. func (c *Config) ParseArtifactMaxSize() int64 { @@ -1959,6 +2107,20 @@ func (c *Config) ParseGradleBuildCacheMaxUploadSize() int64 { return size } +// ParseAPKSnapshots returns how long earlier Alpine APK indexes are kept +// after upstream replaced them. Returns 0 when snapshots are disabled or the +// value is invalid. +func (c *Config) ParseAPKSnapshots() time.Duration { + if c.Upstream.APKSnapshots == "" { + return 0 + } + d, err := parseDayDuration(c.Upstream.APKSnapshots) + if err != nil || d < 0 { + return 0 + } + return d +} + // ParseGradleBuildCacheMaxAge returns age-based eviction threshold. // Returns 0 when disabled or invalid. func (c *Config) ParseGradleBuildCacheMaxAge() time.Duration { @@ -2041,7 +2203,16 @@ func ParseSize(s string) (int64, error) { if err != nil { return 0, fmt.Errorf("invalid number %q", numStr) } - return int64(num * float64(s2.mult)), nil + // The comparisons also reject NaN. + size := num * float64(s2.mult) + if !(size >= 0 && size < math.MaxInt64) { + return 0, fmt.Errorf("invalid size %q: must be between 0 and %d bytes", s, int64(math.MaxInt64)) + } + // max_size 0 means unlimited, so don't let a tiny size become 0. + if num != 0 && int64(size) == 0 { + return 0, fmt.Errorf("invalid size %q: less than one byte", s) + } + return int64(size), nil } } @@ -2050,6 +2221,9 @@ func ParseSize(s string) (int64, error) { if err != nil { return 0, fmt.Errorf("invalid size %q", s) } + if num < 0 { + return 0, fmt.Errorf("invalid size %q: must not be negative", s) + } return num, nil } @@ -2090,7 +2264,3 @@ func (a *AuthConfig) Header() (name, value string) { func expandEnv(s string) string { return os.Expand(s, os.Getenv) } - -func envBool(v string) bool { - return v == "true" || v == "1" -} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index a0132cf..ff8fc50 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -296,6 +297,8 @@ func TestParseSize(t *testing.T) { {"1TB", 1024 * 1024 * 1024 * 1024, false}, {"invalid", 0, true}, {"10XB", 0, true}, + {"0GB", 0, false}, + {"0.0MB", 0, false}, } for _, tt := range tests { @@ -466,8 +469,9 @@ func TestLoadGradleBuildCacheReadOnlyFromEnv(t *testing.T) { {" FALSE ", true, false, 0}, {"True", false, true, 0}, {"1", false, true, 0}, - {"yes", false, true, 1}, - {"off", false, true, 1}, + {"yes", false, true, 0}, + {"off", true, false, 0}, + {"maybe", false, true, 1}, {"", false, false, 0}, } for _, tt := range tests { @@ -785,6 +789,29 @@ func TestParseMetadataMaxSize(t *testing.T) { } } +func TestParseMetadataStreamMaxSize(t *testing.T) { + cfg := Default() + if got := cfg.ParseMetadataStreamMaxSize(); got != defaultMetadataStreamMaxSize { + t.Errorf("unset: %d, want %d", got, defaultMetadataStreamMaxSize) + } + cfg.MetadataStreamMaxSize = "2GB" + if got := cfg.ParseMetadataStreamMaxSize(); got != 2<<30 { + t.Errorf("2GB: %d", got) + } + t.Setenv("PROXY_METADATA_STREAM_MAX_SIZE", "300MB") + cfg = Default() + cfg.LoadFromEnv() + if got := cfg.ParseMetadataStreamMaxSize(); got != 300<<20 { + t.Errorf("from env: %d", got) + } + for _, bad := range []string{"0", "lots"} { + cfg.MetadataStreamMaxSize = bad + if err := cfg.Validate(); err == nil { + t.Errorf("metadata_stream_max_size %q validated", bad) + } + } +} + func TestValidateMetadataMaxSize(t *testing.T) { cfg := Default() cfg.MetadataMaxSize = "not-a-size" @@ -1141,6 +1168,17 @@ func TestLoadMetadataTTLFromEnv(t *testing.T) { } } +func TestLoadChangeFeedsFromEnv(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_UPSTREAM_NPM_CHANGE_FEED", "true") + t.Setenv("PROXY_UPSTREAM_COMPOSER_CHANGE_FEED", "true") + cfg.LoadFromEnv() + + if !cfg.Upstream.NPMChangeFeed || !cfg.Upstream.ComposerChangeFeed { + t.Errorf("change feeds = npm %t, composer %t, want both on", cfg.Upstream.NPMChangeFeed, cfg.Upstream.ComposerChangeFeed) + } +} + func TestParseGradleBuildCacheConfig(t *testing.T) { cfg := Default() @@ -1729,6 +1767,30 @@ func TestValidateGoSumDBs(t *testing.T) { cfg.Upstream.GoSumDBsOffline = true }, }, + { + name: "npm change feed requires metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = false + cfg.Upstream.NPMChangeFeed = true + }, + wantErr: true, + }, + { + name: "Composer change feed requires metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = false + cfg.Upstream.ComposerChangeFeed = true + }, + wantErr: true, + }, + { + name: "change feeds with metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = true + cfg.Upstream.NPMChangeFeed = true + cfg.Upstream.ComposerChangeFeed = true + }, + }, } for _, tt := range tests { @@ -2057,6 +2119,41 @@ upstream: } } +func TestAPKSnapshots(t *testing.T) { + cfg := Default() + if got := cfg.ParseAPKSnapshots(); got != 0 { + t.Errorf("default ParseAPKSnapshots = %v, want 0", got) + } + + for value, want := range map[string]time.Duration{ + "90d": 90 * 24 * time.Hour, + "12h": 12 * time.Hour, + "0": 0, + } { + cfg.Upstream.APKSnapshots = value + if err := cfg.Validate(); err != nil { + t.Errorf("Validate(%q): %v", value, err) + } + if got := cfg.ParseAPKSnapshots(); got != want { + t.Errorf("ParseAPKSnapshots(%q) = %v, want %v", value, got, want) + } + } + + for _, value := range []string{"forever", "-1d"} { + cfg.Upstream.APKSnapshots = value + if err := cfg.Validate(); err == nil { + t.Errorf("Validate(%q) succeeded, want an error", value) + } + } + + cfg = Default() + t.Setenv("PROXY_UPSTREAM_APK_SNAPSHOTS", "30d") + cfg.LoadFromEnv() + if cfg.Upstream.APKSnapshots != "30d" { + t.Errorf("APKSnapshots from env = %q, want 30d", cfg.Upstream.APKSnapshots) + } +} + func TestCooldownShortest(t *testing.T) { for _, tc := range []struct { name string @@ -2076,3 +2173,101 @@ func TestCooldownShortest(t *testing.T) { } } } + +func TestLoadBoolsFromEnvParseStrictly(t *testing.T) { + t.Run("no turns off a setting from the config file", func(t *testing.T) { + t.Setenv("PROXY_TRUST_FORWARDED_FOR", "no") + cfg := Default() + cfg.TrustForwardedFor = true + cfg.LoadFromEnv() + if cfg.TrustForwardedFor { + t.Error("TrustForwardedFor = true, want false") + } + }) + + tests := []struct { + value string + start bool + want bool + warnings int + }{ + {"true", false, true, 0}, + {"True", false, true, 0}, + {" TRUE ", false, true, 0}, + {"1", false, true, 0}, + {"false", true, false, 0}, + {"FALSE", true, false, 0}, + {"0", true, false, 0}, + {"yes", false, true, 0}, + {" On ", false, true, 0}, + {"no", true, false, 0}, + {"OFF", true, false, 0}, + {"maybe", false, false, 1}, + {"ture", true, true, 1}, + } + for _, tt := range tests { + t.Run(tt.value, func(t *testing.T) { + t.Setenv("PROXY_SCANNING_ENABLED", tt.value) + cfg := Default() + cfg.Scanning.Enabled = tt.start + cfg.LoadFromEnv() + if cfg.Scanning.Enabled != tt.want { + t.Errorf("Scanning.Enabled = %v, want %v", cfg.Scanning.Enabled, tt.want) + } + if got := len(cfg.EnvWarnings()); got != tt.warnings { + t.Errorf("len(EnvWarnings()) = %d, want %d: %v", got, tt.warnings, cfg.EnvWarnings()) + } + }) + } +} + +func TestScannerEcosystems(t *testing.T) { + cfg := ScanningConfig{ + Enabled: true, + SigningKey: "s3cret", + Scanners: []ScannerConfig{{ + Name: "clamav", URL: "http://scanner.invalid/scan", + Ecosystems: []string{"go", "Docker", "rubygems", "npm", "arch", "git"}, + }}, + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() error: %v", err) + } + want := []string{"golang", "oci", "gem", "npm", "arch", "git"} + if got := cfg.Scanners[0].Ecosystems; !slices.Equal(got, want) { + t.Errorf("Ecosystems = %v, want %v", got, want) + } + + cfg.Scanners[0].Ecosystems = []string{"npm", "pypy"} + err := cfg.Validate() + if err == nil || !strings.Contains(err.Error(), "pypy") { + t.Errorf("Validate() error = %v, want unknown ecosystem pypy", err) + } +} + +func TestParseSizeRejectsOutOfRange(t *testing.T) { + for _, s := range []string{"-5GB", "-1", "-0.5K", "NaNGB", "InfGB", "+InfB", "1e30GB", "9999999999TB", "0.5B", "0.000000000001GB"} { + if got, err := ParseSize(s); err == nil { + t.Errorf("ParseSize(%q) = %d, want error", s, got) + } + } + cfg := Default() + cfg.Storage.MaxSize = "-5GB" + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted storage.max_size -5GB") + } +} + +func TestValidateRejectsNegativeTTLs(t *testing.T) { + cfg := Default() + cfg.MetadataTTL = "-5m" + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted metadata_ttl -5m") + } + + cfg = Default() + cfg.Storage.DirectServeTTL = "-5m" + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted storage.direct_serve_ttl -5m") + } +} diff --git a/internal/config/cooldown_profiles.go b/internal/config/cooldown_profiles.go new file mode 100644 index 0000000..225dd20 --- /dev/null +++ b/internal/config/cooldown_profiles.go @@ -0,0 +1,64 @@ +package config + +import ( + "fmt" + "maps" + "regexp" + "slices" + + "github.com/git-pkgs/cooldown" +) + +var cooldownProfileName = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) + +// Policy builds the policy for this URL's cooldown settings. +func (c *CooldownConfig) Policy() (*cooldown.Policy, error) { + return cooldown.NewPolicy(&cooldown.Config{ + Default: c.Default, Ecosystems: c.Ecosystems, Packages: c.NormalizedPackages(), + }, c.PackagePatterns) +} + +// ValidateProfiles checks names and policies before they become routes. +func (c *CooldownConfig) ValidateProfiles() error { + for _, name := range slices.Sorted(maps.Keys(c.Profiles)) { + profile := c.Profiles[name] + if !cooldownProfileName.MatchString(name) { + return fmt.Errorf("invalid cooldown profile name %q: use lowercase letters, digits, hyphens or underscores", name) + } + if len(profile.Profiles) > 0 { + return fmt.Errorf("cooldown.profiles.%s cannot contain profiles", name) + } + if err := profile.validateProfileDurations(); err != nil { + return fmt.Errorf("cooldown.profiles.%s: %w", name, err) + } + if _, err := profile.Policy(); err != nil { + return fmt.Errorf("cooldown.profiles.%s: %w", name, err) + } + } + return nil +} + +func (c *CooldownConfig) validateProfileDurations() error { + values := map[string]string{"default": c.Default} + for group, overrides := range map[string]map[string]string{ + "ecosystems": c.Ecosystems, "packages": c.Packages, "package_patterns": c.PackagePatterns, + } { + for key, value := range overrides { + values[group+"."+key] = value + } + } + for _, key := range slices.Sorted(maps.Keys(values)) { + value := values[key] + if key == "default" && value == "" { + continue + } + duration, err := cooldown.ParseDuration(value) + if err != nil { + return fmt.Errorf("invalid %s: %w", key, err) + } + if duration < 0 { + return fmt.Errorf("invalid %s: duration must be non-negative", key) + } + } + return nil +} diff --git a/internal/config/cooldown_profiles_test.go b/internal/config/cooldown_profiles_test.go new file mode 100644 index 0000000..6f10362 --- /dev/null +++ b/internal/config/cooldown_profiles_test.go @@ -0,0 +1,81 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestLoadCooldownProfiles(t *testing.T) { + filename := filepath.Join(t.TempDir(), "config.yaml") + data := `cooldown: + default: 7d + ecosystems: + npm: "0" + profiles: + 3d: + default: 3d + packages: + "pkg:pypi/Django": "0" + strict: + default: 14d + package_patterns: + "pkg:npm/@example/*": 1d +` + if err := os.WriteFile(filename, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(filename) + if err != nil { + t.Fatal(err) + } + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + profile := cfg.Cooldown.Profiles["3d"] + policy, err := profile.Policy() + if err != nil { + t.Fatal(err) + } + if got := policy.For("npm", "pkg:npm/demo"); got != 72*time.Hour { + t.Fatalf("profile inherited global ecosystem override: %v", got) + } + if got := policy.For("pypi", "pkg:pypi/django"); got != 0 { + t.Fatalf("canonical package override = %v", got) + } + strict := cfg.Cooldown.Profiles["strict"] + policy, err = strict.Policy() + if err != nil { + t.Fatal(err) + } + if got := policy.For("npm", "pkg:npm/%40example/tool"); got != 24*time.Hour { + t.Fatalf("profile pattern = %v", got) + } +} + +func TestLoadRejectsInvalidCooldownProfiles(t *testing.T) { + for _, tc := range []struct{ name, profile, want string }{ + {"path", `"a/b": {default: 3d}`, "profile name"}, + {"uppercase", `Strict: {default: 3d}`, "profile name"}, + {"empty", `"": {default: 3d}`, "profile name"}, + {"duration", `strict: {default: nope}`, "cooldown.profiles.strict"}, + {"pattern", `strict: {package_patterns: {"pkg:npm/[": 3d}}`, "cooldown.profiles.strict"}, + {"nested", `strict: {profiles: {nested: {default: 3d}}}`, "cannot contain profiles"}, + } { + t.Run(tc.name, func(t *testing.T) { + filename := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(filename, []byte("cooldown:\n profiles:\n "+tc.profile+"\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(filename) + if err == nil { + err = cfg.Validate() + } + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Load error = %v, want %q", err, tc.want) + } + }) + } +} diff --git a/internal/config/mirror.go b/internal/config/mirror.go index 7f6c668..3b56fd8 100644 --- a/internal/config/mirror.go +++ b/internal/config/mirror.go @@ -15,6 +15,105 @@ type MirrorConfig struct { // Schedules re-mirror container image tags at a fixed interval, so a // moved tag is picked up even when nobody pulls it. Schedules []MirrorSchedule `json:"schedules" yaml:"schedules"` + + // Refresh, when set, keeps the most used packages up to date in the + // background. + Refresh *MirrorRefresh `json:"refresh" yaml:"refresh"` +} + +const ( + defaultMirrorRefreshTop = 50 + defaultMirrorRefreshSince = 14 * 24 * time.Hour +) + +// MirrorRefresh mirrors new releases of the packages clients use most, so +// they are cached before the first client asks for them. +type MirrorRefresh struct { + // Interval is how often the refresh runs, at least one minute (e.g. + // "24h"). + Interval string `json:"interval" yaml:"interval"` + + // Top is how many packages per ecosystem are refreshed, the ones with + // the most downloads among those used within Since. Default: 50. + Top int `json:"top" yaml:"top"` + + // Since is how recently a package must have been downloaded to be + // refreshed (e.g. "14d"). Default: 14d. + Since string `json:"since" yaml:"since"` + + // MaxVersions caps how many new versions one run mirrors. 0 means no + // cap. + MaxVersions int `json:"max_versions" yaml:"max_versions"` + + // MaxBytes caps how much one run downloads (e.g. "2GB"). Empty means no + // cap. + MaxBytes string `json:"max_bytes" yaml:"max_bytes"` + + interval time.Duration + since time.Duration + maxBytes int64 +} + +// IntervalDuration returns the parsed Interval. Validate must run first. +func (r *MirrorRefresh) IntervalDuration() time.Duration { return r.interval } + +// TopPackages returns Top, or its default. +func (r *MirrorRefresh) TopPackages() int { + if r.Top == 0 { + return defaultMirrorRefreshTop + } + return r.Top +} + +// SinceDuration returns the parsed Since, or its default. Validate must run +// first. +func (r *MirrorRefresh) SinceDuration() time.Duration { return r.since } + +// MaxVersionsPerRun returns MaxVersions; 0 means no cap. +func (r *MirrorRefresh) MaxVersionsPerRun() int { return r.MaxVersions } + +// MaxBytesPerRun returns the parsed MaxBytes; 0 means no cap. Validate must +// run first. +func (r *MirrorRefresh) MaxBytesPerRun() int64 { return r.maxBytes } + +func (r *MirrorRefresh) validate() error { + d, err := time.ParseDuration(r.Interval) + if err != nil { + return fmt.Errorf("invalid mirror.refresh.interval %q: %w", r.Interval, err) + } + if d < minMirrorScheduleInterval { + return fmt.Errorf("invalid mirror.refresh.interval %q: must be at least %s", r.Interval, minMirrorScheduleInterval) + } + r.interval = d + if r.Top < 0 { + return fmt.Errorf("invalid mirror.refresh.top %d: must not be negative", r.Top) + } + r.since = defaultMirrorRefreshSince + if r.Since != "" { + since, err := parseDayDuration(r.Since) + if err != nil { + return fmt.Errorf("invalid mirror.refresh.since %q: %w", r.Since, err) + } + if since <= 0 { + return fmt.Errorf("invalid mirror.refresh.since %q: must be positive", r.Since) + } + r.since = since + } + if r.MaxVersions < 0 { + return fmt.Errorf("invalid mirror.refresh.max_versions %d: must not be negative", r.MaxVersions) + } + r.maxBytes = 0 + if r.MaxBytes != "" { + n, err := ParseSize(r.MaxBytes) + if err != nil { + return fmt.Errorf("invalid mirror.refresh.max_bytes %q: %w", r.MaxBytes, err) + } + if n < 0 { + return fmt.Errorf("invalid mirror.refresh.max_bytes %q: must not be negative", r.MaxBytes) + } + r.maxBytes = n + } + return nil } // MirrorSchedule is a set of image PURLs mirrored every Interval. @@ -38,6 +137,16 @@ func (s *MirrorSchedule) IntervalDuration() time.Duration { return s.interval } +// Warnings returns settings that are valid but do less than they may be +// expected to. The server logs them at startup. +func (c *Config) Warnings() []string { + var warnings []string + if c.Mirror.Refresh != nil && !c.CacheMetadata { + warnings = append(warnings, "mirror.refresh without cache_metadata does not refresh Debian, Alpine or RPM packages: their indexes are found through the metadata cache") + } + return warnings +} + // Validate checks every schedule. func (m *MirrorConfig) Validate() error { for i := range m.Schedules { @@ -45,6 +154,9 @@ func (m *MirrorConfig) Validate() error { return err } } + if m.Refresh != nil { + return m.Refresh.validate() + } return nil } diff --git a/internal/config/mirror_test.go b/internal/config/mirror_test.go index ae1220d..d5dff5f 100644 --- a/internal/config/mirror_test.go +++ b/internal/config/mirror_test.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "strings" "testing" "time" ) @@ -87,3 +88,101 @@ mirror: t.Error("Validate() accepted a 30s interval in the second schedule") } } + +func TestMirrorRefreshValidate(t *testing.T) { + tests := []struct { + name string + refresh *MirrorRefresh + wantErr bool + }{ + {name: "defaults", refresh: &MirrorRefresh{Interval: "24h"}}, + {name: "all fields", refresh: &MirrorRefresh{Interval: "12h", Top: 20, Since: "14d", MaxVersions: 100, MaxBytes: "2GB"}}, + {name: "missing interval", refresh: &MirrorRefresh{}, wantErr: true}, + {name: "interval under a minute", refresh: &MirrorRefresh{Interval: "10s"}, wantErr: true}, + {name: "negative top", refresh: &MirrorRefresh{Interval: "1h", Top: -1}, wantErr: true}, + {name: "bad since", refresh: &MirrorRefresh{Interval: "1h", Since: "recently"}, wantErr: true}, + {name: "negative since", refresh: &MirrorRefresh{Interval: "1h", Since: "-1d"}, wantErr: true}, + {name: "negative max_versions", refresh: &MirrorRefresh{Interval: "1h", MaxVersions: -1}, wantErr: true}, + {name: "bad max_bytes", refresh: &MirrorRefresh{Interval: "1h", MaxBytes: "lots"}, wantErr: true}, + {name: "negative max_bytes", refresh: &MirrorRefresh{Interval: "1h", MaxBytes: "-1GB"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := MirrorConfig{Refresh: tt.refresh} + err := m.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestMirrorRefreshDefaults(t *testing.T) { + m := MirrorConfig{Refresh: &MirrorRefresh{Interval: "24h"}} + if err := m.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + r := m.Refresh + if r.IntervalDuration() != 24*time.Hour || r.TopPackages() != 50 || r.SinceDuration() != 14*24*time.Hour || + r.MaxVersionsPerRun() != 0 || r.MaxBytesPerRun() != 0 { + t.Errorf("got interval %v top %d since %v max_versions %d max_bytes %d, want 24h 50 336h 0 0", + r.IntervalDuration(), r.TopPackages(), r.SinceDuration(), r.MaxVersionsPerRun(), r.MaxBytesPerRun()) + } +} + +func TestValidateRejectsMirrorRefreshWithoutCachedArtifacts(t *testing.T) { + cfg := Default() + cfg.Mirror.Refresh = &MirrorRefresh{Interval: "24h"} + cfg.Storage.CacheArtifacts = false + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted mirror.refresh with storage.cache_artifacts: false") + } +} + +func TestLoadYAMLMirrorRefresh(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + data := ` +mirror: + refresh: + interval: 12h + top: 10 + since: 7d + max_versions: 200 + max_bytes: 1GB +` + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + r := cfg.Mirror.Refresh + if r == nil { + t.Fatal("mirror.refresh not loaded") + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if r.IntervalDuration() != 12*time.Hour || r.TopPackages() != 10 || r.SinceDuration() != 7*24*time.Hour || + r.MaxVersionsPerRun() != 200 || r.MaxBytesPerRun() != 1<<30 { + t.Errorf("got interval %v top %d since %v max_versions %d max_bytes %d", + r.IntervalDuration(), r.TopPackages(), r.SinceDuration(), r.MaxVersionsPerRun(), r.MaxBytesPerRun()) + } +} + +func TestWarningsMirrorRefreshWithoutCacheMetadata(t *testing.T) { + cfg := Default() + cfg.Mirror.Refresh = &MirrorRefresh{Interval: "24h"} + cfg.CacheMetadata = false + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() = %v, want mirror.refresh accepted for the other ecosystems", err) + } + if got := cfg.Warnings(); len(got) != 1 || !strings.Contains(got[0], "cache_metadata") { + t.Errorf("Warnings() = %q, want one about cache_metadata", got) + } + cfg.CacheMetadata = true + if got := cfg.Warnings(); len(got) != 0 { + t.Errorf("Warnings() with cache_metadata = %q, want none", got) + } +} diff --git a/internal/config/projects.go b/internal/config/projects.go new file mode 100644 index 0000000..f416007 --- /dev/null +++ b/internal/config/projects.go @@ -0,0 +1,53 @@ +package config + +import ( + "fmt" + "time" +) + +// ProjectsConfig configures the project inventory: repositories upload an +// SBOM to /api/projects, and the proxy shows which of them use which package +// versions. +type ProjectsConfig struct { + // Enabled registers the /api/projects endpoints and the UI pages. Off by + // default. + Enabled bool `json:"enabled" yaml:"enabled"` + + // Token is the bearer token required to upload and delete projects. + // Required when Enabled. Supports ${VAR_NAME} expansion like AuthConfig + // fields. + Token string `json:"token" yaml:"token"` + + // MaxAge removes projects that have not uploaded an SBOM for this long + // (e.g., "90d"). Empty or "0" keeps them until they are deleted. + MaxAge string `json:"max_age" yaml:"max_age"` + + maxAge time.Duration +} + +// TokenExpanded returns Token with ${VAR_NAME} references expanded. +func (p *ProjectsConfig) TokenExpanded() string { + return expandEnv(p.Token) +} + +// MaxAgeDuration returns the parsed MaxAge, 0 to keep projects. Validate +// must run first. +func (p *ProjectsConfig) MaxAgeDuration() time.Duration { + return p.maxAge +} + +// Validate checks the project inventory settings. +func (p *ProjectsConfig) Validate() error { + p.maxAge = 0 + if p.MaxAge != "" { + d, err := parseRetentionDuration("projects.max_age", p.MaxAge) + if err != nil { + return err + } + p.maxAge = d + } + if p.Enabled && p.TokenExpanded() == "" { + return fmt.Errorf("projects.token is required when projects.enabled is true") + } + return nil +} diff --git a/internal/config/projects_test.go b/internal/config/projects_test.go new file mode 100644 index 0000000..ce3c27c --- /dev/null +++ b/internal/config/projects_test.go @@ -0,0 +1,69 @@ +package config + +import ( + "testing" + "time" +) + +func TestProjectsConfigValidate(t *testing.T) { + t.Setenv("PROJECTS_TEST_TOKEN", "from-env") + + tests := []struct { + name string + projects ProjectsConfig + wantErr bool + maxAge time.Duration + }{ + {name: "disabled without token", projects: ProjectsConfig{}}, + {name: "enabled without token", projects: ProjectsConfig{Enabled: true}, wantErr: true}, + {name: "enabled with token", projects: ProjectsConfig{Enabled: true, Token: "s3cret"}}, + {name: "token from env reference", projects: ProjectsConfig{Enabled: true, Token: "${PROJECTS_TEST_TOKEN}"}}, + {name: "token env reference unset", projects: ProjectsConfig{Enabled: true, Token: "${PROJECTS_TEST_UNSET}"}, wantErr: true}, + {name: "max age in days", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "90d"}, maxAge: 90 * 24 * time.Hour}, + {name: "max age in hours", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "36h"}, maxAge: 36 * time.Hour}, + {name: "max age zero keeps projects", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "0"}}, + {name: "invalid max age", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "soon"}, wantErr: true}, + {name: "negative max age", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "-1d"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.projects.Validate() + if (err != nil) != tt.wantErr { + t.Fatalf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + if got := tt.projects.MaxAgeDuration(); err == nil && got != tt.maxAge { + t.Errorf("MaxAgeDuration() = %v, want %v", got, tt.maxAge) + } + }) + } +} + +func TestLoadFromEnvProjects(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_PROJECTS_ENABLED", "true") + t.Setenv("PROXY_PROJECTS_TOKEN", "env-token") + t.Setenv("PROXY_PROJECTS_MAX_AGE", "30d") + + cfg.LoadFromEnv() + + if !cfg.Projects.Enabled { + t.Error("Projects.Enabled not loaded from PROXY_PROJECTS_ENABLED") + } + if cfg.Projects.TokenExpanded() != "env-token" { + t.Errorf("Projects.Token = %q, want env-token", cfg.Projects.Token) + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := cfg.Projects.MaxAgeDuration(); got != 30*24*time.Hour { + t.Errorf("MaxAgeDuration() = %v, want 30 days", got) + } +} + +func TestValidateRejectsProjectsWithoutToken(t *testing.T) { + cfg := Default() + cfg.Projects = ProjectsConfig{Enabled: true} + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted projects.enabled without a token") + } +} diff --git a/internal/config/retention.go b/internal/config/retention.go index 9493d28..ee78560 100644 --- a/internal/config/retention.go +++ b/internal/config/retention.go @@ -67,6 +67,11 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error } } + if r.KeepLatest < 0 { + return retention.Rules{}, fmt.Errorf("invalid storage.retention.keep_latest: must not be negative") + } + rules.KeepLatest = r.KeepLatest + for _, key := range sortedKeys(r.Ecosystems) { field := "storage.retention.ecosystems." + key if !retention.IsKnown(key) { diff --git a/internal/config/retention_test.go b/internal/config/retention_test.go index eb614e8..9eb55e7 100644 --- a/internal/config/retention_test.go +++ b/internal/config/retention_test.go @@ -1,6 +1,8 @@ package config import ( + "os" + "path/filepath" "strings" "testing" "time" @@ -69,6 +71,34 @@ func TestRetentionRulesResolves(t *testing.T) { } } +// keep_latest is read from the config file and carried into the rules. +func TestRetentionKeepLatest(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + content := "storage:\n retention:\n default: \"30d\"\n keep_latest: 2\n" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + rules, err := cfg.RetentionRules(retentionTestRegistry()) + if err != nil { + t.Fatalf("RetentionRules() error = %v", err) + } + if rules.KeepLatest != 2 { + t.Errorf("KeepLatest = %d, want 2", rules.KeepLatest) + } + + rules, err = Default().RetentionRules(retentionTestRegistry()) + if err != nil { + t.Fatalf("RetentionRules() error = %v", err) + } + if rules.KeepLatest != 0 { + t.Errorf("KeepLatest = %d by default, want 0", rules.KeepLatest) + } +} + func TestRetentionRulesErrors(t *testing.T) { tests := []struct { name string @@ -76,6 +106,7 @@ func TestRetentionRulesErrors(t *testing.T) { wantErr string }{ {"bad default", func(r *RetentionConfig) { r.Default = "7x" }, "storage.retention.default"}, + {"negative keep_latest", func(r *RetentionConfig) { r.KeepLatest = -1 }, "storage.retention.keep_latest: must not be negative"}, {"negative default", func(r *RetentionConfig) { r.Default = "-1d" }, "must not be negative"}, {"NaN days", func(r *RetentionConfig) { r.Default = "NaNd" }, "out of range"}, {"infinite days", func(r *RetentionConfig) { r.Default = "Infd" }, "out of range"}, diff --git a/internal/database/analytics_test.go b/internal/database/analytics_test.go index 269b9d2..5084acd 100644 --- a/internal/database/analytics_test.go +++ b/internal/database/analytics_test.go @@ -373,3 +373,92 @@ func TestMergeAliasedEcosystemsBreaksSizeTiesByName(t *testing.T) { t.Errorf("input order changed the surviving spelling: %q vs %q", first, second) } } + +// Cooldown stores publish times of versions nobody downloaded, without a +// package row. They count under the ecosystem of their package PURL, not as +// unattributed. +func TestGetEcosystemStatsAttributesPublishTimesByPURL(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) + published := time.Now().Add(-time.Hour) + for _, v := range []struct{ version, pkg string }{ + {"pkg:npm/react@19.0.0", "pkg:npm/react"}, + {"pkg:golang/github.com/swaggo/swag@v1.16.3", "pkg:golang/github.com/swaggo/swag"}, + {"pkg:golang/github.com/swaggo/swag@v1.16.4", "pkg:golang/github.com/swaggo/swag"}, + {"pkg:composer/vendor/pkg@1.0.0", "pkg:composer/vendor/pkg"}, + {"not-a-purl@1", "not-a-purl"}, + } { + if err := db.SetVersionPublishedAt(v.version, v.pkg, published); err != nil { + t.Fatal(err) + } + } + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + versions := map[string]int64{} + for _, e := range stats { + versions[e.Ecosystem] = e.Versions + } + want := map[string]int64{"npm": 2, "golang": 2, "composer": 1, unattributedEcosystem: 1} + for ecosystem, n := range want { + if versions[ecosystem] != n { + t.Errorf("versions of %s = %d, want %d (all: %v)", ecosystem, versions[ecosystem], n, versions) + } + } + if len(versions) != len(want) { + t.Errorf("ecosystems = %v, want %v", versions, want) + } + }) +} + +// TestGetEcosystemStatsVersionsReconcile checks that the versions counted +// per ecosystem add up to the cache stats' total, with publish times stored +// under namespaced PURLs, an alias of a package row's ecosystem, and a PURL +// that can't be parsed. +func TestGetEcosystemStatsVersionsReconcile(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) + seedArtifact(t, db, "go", "github.com/pkg/errors", "v0.9.1", 500, 1) + published := time.Now().Add(-time.Hour) + for _, v := range []struct{ version, pkg string }{ + {"pkg:npm/%40babel/core@7.25.2", "pkg:npm/%40babel/core"}, + {"pkg:maven/org.apache.commons/commons-lang3@3.14.0", "pkg:maven/org.apache.commons/commons-lang3"}, + {"pkg:pypi/requests@2.32.3", "pkg:pypi/requests"}, + {"pkg:golang/github.com/swaggo/swag@v1.16.4", "pkg:golang/github.com/swaggo/swag"}, + {"~odd@1", "~odd"}, + } { + if err := db.SetVersionPublishedAt(v.version, v.pkg, published); err != nil { + t.Fatal(err) + } + } + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + versions := map[string]int64{} + var sum int64 + for _, e := range stats { + versions[e.Ecosystem] += e.Versions + sum += e.Versions + } + want := map[string]int64{"npm": 2, "go": 2, "maven": 1, "pypi": 1, unattributedEcosystem: 1} + if len(versions) != len(want) { + t.Errorf("ecosystems = %v, want %v", versions, want) + } + for ecosystem, n := range want { + if versions[ecosystem] != n { + t.Errorf("versions of %s = %d, want %d (all: %v)", ecosystem, versions[ecosystem], n, versions) + } + } + cache, err := db.GetCacheStats() + if err != nil { + t.Fatal(err) + } + if sum != cache.TotalVersions { + t.Errorf("versions per ecosystem add up to %d, cache stats count %d", sum, cache.TotalVersions) + } + }) +} diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a6..500ad0d 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB { // Drop and recreate every table CreateSchema creates for clean test state; // leftover migration records make the next CreateSchema fail on the // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "migrations", "schema_info"} + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "metadata_snapshots", "projects", "project_components", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } @@ -1164,3 +1164,86 @@ func TestVersionPublishedAtPreserved(t *testing.T) { } }) } + +func TestGetFetchedVersionPURLs(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkg = "pkg:npm/leftpad" + for _, v := range []string{"1.0.0", "2.0.0", "3.0.0"} { + if err := db.UpsertVersion(&Version{PURL: pkg + "@" + v, PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + } + if err := db.UpsertVersion(&Version{PURL: "pkg:npm/other@1.0.0", PackagePURL: "pkg:npm/other"}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + artifacts := []Artifact{ + {VersionPURL: pkg + "@1.0.0", Filename: "leftpad-1.0.0.tgz", UpstreamURL: "https://example.com/1", + StoragePath: sql.NullString{String: "npm/leftpad-1.0.0.tgz", Valid: true}}, + // Evicted: the row stays without a storage path. + {VersionPURL: pkg + "@2.0.0", Filename: "leftpad-2.0.0.tgz", UpstreamURL: "https://example.com/2"}, + {VersionPURL: "pkg:npm/other@1.0.0", Filename: "other-1.0.0.tgz", UpstreamURL: "https://example.com/3"}, + } + for i := range artifacts { + if err := db.UpsertArtifact(&artifacts[i]); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + } + + got, err := db.GetFetchedVersionPURLs(pkg) + if err != nil { + t.Fatalf("GetFetchedVersionPURLs: %v", err) + } + want := map[string]bool{pkg + "@1.0.0": true, pkg + "@2.0.0": true} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for purl := range want { + if !got[purl] { + t.Errorf("missing %s in %v", purl, got) + } + } + }) +} + +// Publish times are written in UTC whatever zone the caller's time is in, so +// a TIMESTAMP column without a zone reads back the same instant. +func TestVersionPublishedAtIsUTC(t *testing.T) { + tokyo, err := time.LoadLocation("Asia/Tokyo") + if err != nil { + t.Skipf("time zone data unavailable: %v", err) + } + prev := time.Local + time.Local = tokyo + t.Cleanup(func() { time.Local = prev }) + + runWithBothDatabases(t, func(t *testing.T, db *DB) { + written := time.Date(2026, 10, 10, 21, 0, 1, 0, tokyo) + want := written.UTC() + check := func(purl string) { + t.Helper() + got, err := db.GetVersionByPURL(purl) + if err != nil || got == nil || !got.PublishedAt.Valid { + t.Fatalf("GetVersionByPURL(%s) = %+v, %v", purl, got, err) + } + // Postgres reads TIMESTAMP back as a fixed +0000 zone rather + // than time.UTC, so compare the offset. + if _, offset := got.PublishedAt.Time.Zone(); !got.PublishedAt.Time.Equal(want) || offset != 0 { + t.Errorf("%s PublishedAt read back as %v, want %v", purl, got.PublishedAt.Time, want) + } + } + + if err := db.SetVersionPublishedAt("pkg:cargo/demo@1.0.0", "pkg:cargo/demo", written); err != nil { + t.Fatal(err) + } + check("pkg:cargo/demo@1.0.0") + + if err := db.UpsertVersion(&Version{ + PURL: "pkg:cargo/demo@2.0.0", + PackagePURL: "pkg:cargo/demo", + PublishedAt: sql.NullTime{Time: written, Valid: true}, + }); err != nil { + t.Fatal(err) + } + check("pkg:cargo/demo@2.0.0") + }) +} diff --git a/internal/database/metadata_cache_test.go b/internal/database/metadata_cache_test.go index 0e19f6c..ee16c25 100644 --- a/internal/database/metadata_cache_test.go +++ b/internal/database/metadata_cache_test.go @@ -2,7 +2,9 @@ package database import ( "database/sql" + "fmt" "path/filepath" + "slices" "testing" "time" ) @@ -448,3 +450,91 @@ func TestRefreshMetadataCache(t *testing.T) { t.Errorf("refresh of a missing row = %v, %v, want false", refreshed, err) } } + +// TestExpireMetadataCache checks that expiring clears fetched_at only on the +// named rows of the ecosystem fetched before the cutoff, and keeps their +// documents and validators. +func TestExpireMetadataCache(t *testing.T) { + db := setupMetadataCacheDB(t) + cutoff := time.Now().UTC().Truncate(time.Second) + rows := []struct { + ecosystem, name string + fetched time.Time + }{ + {"composer", "acme/lib", cutoff.Add(-time.Minute)}, + {"composer", "acme/lib~dev", cutoff.Add(-time.Minute)}, + {"composer", "acme/new", cutoff.Add(time.Second)}, + {"composer", "acme/other", cutoff.Add(-time.Minute)}, + {"composer", "routes/abc/acme/lib", cutoff.Add(-time.Minute)}, + {testEcosystemNPM, "acme/lib", cutoff.Add(-time.Minute)}, + } + for _, r := range rows { + err := db.UpsertMetadataCache(&MetadataCacheEntry{ + Ecosystem: r.ecosystem, + Name: r.name, + StoragePath: "_metadata/" + r.ecosystem + "/" + r.name, + ETag: sql.NullString{String: `"abc"`, Valid: true}, + ContentDigest: sql.NullString{String: "sha256:aaaa", Valid: true}, + FetchedAt: sql.NullTime{Time: r.fetched, Valid: true}, + }) + if err != nil { + t.Fatal(err) + } + } + + expired, err := db.ExpireMetadataCache("composer", []string{"acme/lib", "acme/lib~dev", "acme/new", "acme/missing"}, cutoff) + if err != nil { + t.Fatal(err) + } + slices.Sort(expired) + if want := []string{"acme/lib", "acme/lib~dev"}; !slices.Equal(expired, want) { + t.Errorf("expired = %v, want %v", expired, want) + } + got, err := db.GetMetadataCache("composer", "acme/lib") + if err != nil { + t.Fatal(err) + } + if got.FetchedAt.Valid || got.ETag.String != `"abc"` || got.ContentDigest.String != "sha256:aaaa" { + t.Errorf("expired row = %+v, want no fetched_at and the same validators", got) + } + for _, r := range []struct{ ecosystem, name string }{{"composer", "acme/new"}, {"composer", "acme/other"}, {testEcosystemNPM, "acme/lib"}} { + if got, _ := db.GetMetadataCache(r.ecosystem, r.name); !got.FetchedAt.Valid { + t.Errorf("%s %s expired, want it left alone", r.ecosystem, r.name) + } + } + + expired, err = db.ExpireAllMetadataCache("composer", cutoff) + if err != nil { + t.Fatal(err) + } + if want := []string{"acme/other"}; !slices.Equal(expired, want) { + t.Errorf("expired all = %v, want %v (routed and newer rows kept)", expired, want) + } +} + +// TestExpireMetadataCacheBatches checks that more names than one statement +// takes are all expired. +func TestExpireMetadataCacheBatches(t *testing.T) { + db := setupMetadataCacheDB(t) + cutoff := time.Now().UTC() + names := make([]string, expireMetadataBatch+2) + for i := range names { + names[i] = fmt.Sprintf("pkg-%d", i) + } + for _, name := range []string{names[0], names[len(names)-1]} { + err := db.UpsertMetadataCache(&MetadataCacheEntry{ + Ecosystem: testEcosystemNPM, Name: name, StoragePath: "_metadata/npm/" + name, + FetchedAt: sql.NullTime{Time: cutoff.Add(-time.Minute), Valid: true}, + }) + if err != nil { + t.Fatal(err) + } + } + expired, err := db.ExpireMetadataCache(testEcosystemNPM, names, cutoff) + if err != nil { + t.Fatal(err) + } + if len(expired) != 2 { + t.Errorf("expired = %v, want both stored names", expired) + } +} diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go new file mode 100644 index 0000000..268baf7 --- /dev/null +++ b/internal/database/metadata_snapshots.go @@ -0,0 +1,156 @@ +package database + +import ( + "database/sql" + "fmt" + "math" + "time" +) + +// MetadataSnapshot is one kept version of a metadata document, so a request +// for a past date can be answered with the document that was current then. +// It is valid from ValidFromMS (Unix milliseconds, the time the proxy first +// stored it) until the next snapshot of the same ecosystem and name. +type MetadataSnapshot struct { + ID int64 `db:"id"` + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + StoragePath string `db:"storage_path"` + ContentType sql.NullString `db:"content_type"` + ContentEncoding sql.NullString `db:"content_encoding"` + ContentDigest string `db:"content_digest"` + Size int64 `db:"size"` + ValidFromMS int64 `db:"valid_from_ms"` +} + +// ValidFrom returns the time the snapshot became current. +func (s *MetadataSnapshot) ValidFrom() time.Time { + return time.UnixMilli(s.ValidFromMS) +} + +const metadataSnapshotColumns = `id, ecosystem, name, storage_path, content_type, content_encoding, + content_digest, size, valid_from_ms` + +// MetadataSnapshotAt returns the snapshot of ecosystem and name that was +// current at at: the newest one that became current before it. It returns nil +// when there is none. +func (db *DB) MetadataSnapshotAt(ecosystem, name string, at time.Time) (*MetadataSnapshot, error) { + return db.metadataSnapshotBefore(ecosystem, name, at.UnixMilli()) +} + +// LatestMetadataSnapshot returns the newest snapshot of ecosystem and name, +// or nil when there is none. +func (db *DB) LatestMetadataSnapshot(ecosystem, name string) (*MetadataSnapshot, error) { + return db.metadataSnapshotBefore(ecosystem, name, math.MaxInt64) +} + +func (db *DB) metadataSnapshotBefore(ecosystem, name string, beforeMS int64) (*MetadataSnapshot, error) { + var snap MetadataSnapshot + err := db.Get(&snap, db.Rebind(`SELECT `+metadataSnapshotColumns+` + FROM metadata_snapshots + WHERE ecosystem = ? AND name = ? AND valid_from_ms < ? + ORDER BY valid_from_ms DESC, id DESC + LIMIT 1`), ecosystem, name, beforeMS) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading metadata snapshot: %w", err) + } + return &snap, nil +} + +// AddMetadataSnapshot records snap as the version of its document current +// from snap.ValidFromMS, unless the snapshot current at that time already has +// its digest, and reports whether it did. On Postgres it first takes an +// advisory lock on the document, so two replicas can't both read the previous +// digest and both insert. The casts let Postgres type the parameters of the +// INSERT ... SELECT. +func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, fmt.Errorf("recording metadata snapshot: %w", err) + } + defer func() { _ = tx.Rollback() }() + if db.dialect == DialectPostgres { + if _, err := tx.Exec("SELECT pg_advisory_xact_lock($1)", metadataSnapshotLockKey(snap.Ecosystem, snap.Name)); err != nil { + return false, fmt.Errorf("locking metadata snapshots: %w", err) + } + } + res, err := tx.Exec(db.Rebind(` + INSERT INTO metadata_snapshots (ecosystem, name, storage_path, content_type, content_encoding, + content_digest, size, valid_from_ms) + SELECT CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), + CAST(? AS TEXT), CAST(? AS BIGINT), CAST(? AS BIGINT) + WHERE COALESCE(( + SELECT content_digest FROM metadata_snapshots + WHERE ecosystem = ? AND name = ? AND valid_from_ms <= CAST(? AS BIGINT) + ORDER BY valid_from_ms DESC, id DESC + LIMIT 1 + ), '') <> CAST(? AS TEXT) + `), snap.Ecosystem, snap.Name, snap.StoragePath, snap.ContentType, snap.ContentEncoding, + snap.ContentDigest, snap.Size, snap.ValidFromMS, + snap.Ecosystem, snap.Name, snap.ValidFromMS, snap.ContentDigest) + if err != nil { + return false, fmt.Errorf("recording metadata snapshot: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("recording metadata snapshot: %w", err) + } + return n > 0, nil +} + +// metadataSnapshotLockKey is the advisory lock AddMetadataSnapshot takes for +// a document. +func metadataSnapshotLockKey(ecosystem, name string) int64 { + return lockKey("metadata-snapshot:" + ecosystem + "\x00" + name) +} + +// PruneMetadataSnapshots deletes the snapshots of ecosystem that stopped +// being current before cutoff, that is those replaced by a newer snapshot +// that became current before cutoff, and queues their blobs for deletion. The +// newest snapshot of each document is always kept. It returns the number of +// snapshots deleted. +func (db *DB) PruneMetadataSnapshots(ecosystem string, cutoff time.Time) (int, error) { + tx, err := db.Beginx() + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + + var expired []struct { + ID int64 `db:"id"` + StoragePath string `db:"storage_path"` + } + err = tx.Select(&expired, db.Rebind(` + SELECT s.id, s.storage_path FROM metadata_snapshots s + WHERE s.ecosystem = ? + AND EXISTS ( + SELECT 1 FROM metadata_snapshots n + WHERE n.ecosystem = s.ecosystem AND n.name = s.name + AND (n.valid_from_ms > s.valid_from_ms OR (n.valid_from_ms = s.valid_from_ms AND n.id > s.id)) + AND n.valid_from_ms < ? + ) + `), ecosystem, cutoff.UnixMilli()) + if err != nil { + return 0, fmt.Errorf("listing expired metadata snapshots: %w", err) + } + + queuedAt := time.Now().UTC() + for _, row := range expired { + if _, err := tx.Exec(db.Rebind(`DELETE FROM metadata_snapshots WHERE id = ?`), row.ID); err != nil { + return 0, fmt.Errorf("deleting metadata snapshot: %w", err) + } + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), row.StoragePath, queuedAt); err != nil { + return 0, fmt.Errorf("queueing metadata snapshot blob: %w", err) + } + } + if err := tx.Commit(); err != nil { + return 0, err + } + return len(expired), nil +} diff --git a/internal/database/metadata_snapshots_test.go b/internal/database/metadata_snapshots_test.go new file mode 100644 index 0000000..71865b9 --- /dev/null +++ b/internal/database/metadata_snapshots_test.go @@ -0,0 +1,190 @@ +package database + +import ( + "testing" + "time" +) + +func addSnapshot(t *testing.T, db *DB, name, digest string, validFrom time.Time) bool { + t.Helper() + added, err := db.AddMetadataSnapshot(&MetadataSnapshot{ + Ecosystem: "alpine", + Name: name, + StoragePath: "_snapshots/alpine/" + name + "/" + digest, + ContentDigest: digest, + Size: 1, + ValidFromMS: validFrom.UnixMilli(), + }) + if err != nil { + t.Fatalf("AddMetadataSnapshot: %v", err) + } + return added +} + +func snapshotDigestAt(t *testing.T, db *DB, name string, at time.Time) string { + t.Helper() + snap, err := db.MetadataSnapshotAt("alpine", name, at) + if err != nil { + t.Fatalf("MetadataSnapshotAt: %v", err) + } + if snap == nil { + return "" + } + return snap.ContentDigest +} + +func TestMetadataSnapshots_AddAndLookup(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + day := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + if !addSnapshot(t, db, "idx", "sha256:a", day) { + t.Fatal("first snapshot not added") + } + if addSnapshot(t, db, "idx", "sha256:a", day.Add(time.Hour)) { + t.Error("unchanged digest added again") + } + if !addSnapshot(t, db, "idx", "sha256:b", day.Add(48*time.Hour)) { + t.Fatal("changed digest not added") + } + // A rollback to an earlier version is a new snapshot. + if !addSnapshot(t, db, "idx", "sha256:a", day.Add(96*time.Hour)) { + t.Fatal("rolled back digest not added") + } + + for _, tc := range []struct { + at time.Time + want string + }{ + {day, ""}, + {day.Add(time.Minute), "sha256:a"}, + {day.Add(72 * time.Hour), "sha256:b"}, + {day.Add(100 * time.Hour), "sha256:a"}, + } { + if got := snapshotDigestAt(t, db, "idx", tc.at); got != tc.want { + t.Errorf("snapshot at %s = %q, want %q", tc.at, got, tc.want) + } + } + latest, err := db.LatestMetadataSnapshot("alpine", "idx") + if err != nil || latest == nil || !latest.ValidFrom().Equal(day.Add(96*time.Hour)) { + t.Errorf("LatestMetadataSnapshot = %+v, %v", latest, err) + } + }) +} + +// TestMetadataSnapshots_RecordedLate adds a version after a newer one was +// recorded, as a request that fetched it first but recorded it last does. It +// is compared with the version current at its own time, so it keeps its place +// in the history. +func TestMetadataSnapshots_RecordedLate(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + day := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + addSnapshot(t, db, "idx", "sha256:a", day) + addSnapshot(t, db, "idx", "sha256:c", day.Add(2*time.Hour)) + if !addSnapshot(t, db, "idx", "sha256:b", day.Add(time.Hour)) { + t.Fatal("version recorded after a newer one not added") + } + if addSnapshot(t, db, "idx", "sha256:b", day.Add(90*time.Minute)) { + t.Error("version already current at that time added again") + } + for _, tc := range []struct { + at time.Time + want string + }{ + {day.Add(30 * time.Minute), "sha256:a"}, + {day.Add(100 * time.Minute), "sha256:b"}, + {day.Add(3 * time.Hour), "sha256:c"}, + } { + if got := snapshotDigestAt(t, db, "idx", tc.at); got != tc.want { + t.Errorf("snapshot at %s = %q, want %q", tc.at, got, tc.want) + } + } + }) +} + +func TestMetadataSnapshots_Prune(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + addSnapshot(t, db, "idx", "sha256:a", now.AddDate(0, 0, -100)) + addSnapshot(t, db, "idx", "sha256:b", now.AddDate(0, 0, -60)) + addSnapshot(t, db, "idx", "sha256:c", now.AddDate(0, 0, -10)) + // The only version of another index stays however old it is. + addSnapshot(t, db, "other", "sha256:x", now.AddDate(0, 0, -200)) + + pruned, err := db.PruneMetadataSnapshots("alpine", now.AddDate(0, 0, -30)) + if err != nil { + t.Fatalf("PruneMetadataSnapshots: %v", err) + } + // a was replaced 60 days ago, b only 10 days ago. + if pruned != 1 { + t.Errorf("pruned = %d, want 1", pruned) + } + if got := snapshotDigestAt(t, db, "idx", now.AddDate(0, 0, -90)); got != "" { + t.Errorf("snapshot of a pruned day = %q, want none", got) + } + if got := snapshotDigestAt(t, db, "idx", now.AddDate(0, 0, -50)); got != "sha256:b" { + t.Errorf("snapshot 50 days ago = %q, want sha256:b", got) + } + if got := snapshotDigestAt(t, db, "other", now); got != "sha256:x" { + t.Errorf("only snapshot of other = %q, want sha256:x", got) + } + + due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if len(due) != 1 || due[0] != "_snapshots/alpine/idx/sha256:a" { + t.Errorf("queued deletes = %v, want the pruned blob", due) + } + }) +} + +// TestMetadataSnapshots_ConcurrentPostgres records a version while another +// replica's transaction has recorded the same one and not committed yet, as +// two replicas can. The second must wait and then find it, so one row +// remains. Without the lock both inserts succeed. +func TestMetadataSnapshots_ConcurrentPostgres(t *testing.T) { + db := createTestPostgresDB(t) + if db == nil { + t.Skip("PROXY_DATABASE_URL not set, skipping postgres test") + } + defer func() { _ = db.Close() }() + const name = "v3.22/main/x86_64/APKINDEX.tar.gz" + + other, err := db.Beginx() + if err != nil { + t.Fatal(err) + } + defer func() { _ = other.Rollback() }() + if _, err := other.Exec("SELECT pg_advisory_xact_lock($1)", metadataSnapshotLockKey("alpine", name)); err != nil { + t.Fatal(err) + } + if _, err := other.Exec(`INSERT INTO metadata_snapshots (ecosystem, name, storage_path, content_digest, size, valid_from_ms) + VALUES ('alpine', $1, 'other', 'sha256:a', 1, $2)`, name, time.Now().UnixMilli()); err != nil { + t.Fatal(err) + } + + done := make(chan bool, 1) + go func() { done <- addSnapshot(t, db, name, "sha256:a", time.Now()) }() + select { + case added := <-done: + t.Fatalf("recorded without waiting for the other replica (added=%v)", added) + case <-time.After(200 * time.Millisecond): + } + if err := other.Commit(); err != nil { + t.Fatal(err) + } + if added := <-done; added { + t.Error("recorded a version the other replica had just recorded") + } + var n int + if err := db.Get(&n, `SELECT COUNT(*) FROM metadata_snapshots WHERE name = $1`, name); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("%d snapshot rows, want 1", n) + } + + // Going back to an earlier digest is still recorded. + if !addSnapshot(t, db, name, "sha256:b", time.Now()) || !addSnapshot(t, db, name, "sha256:a", time.Now().Add(time.Millisecond)) { + t.Error("a change back to an earlier digest was not recorded") + } +} diff --git a/internal/database/metadata_upstream_url_test.go b/internal/database/metadata_upstream_url_test.go new file mode 100644 index 0000000..361a2cc --- /dev/null +++ b/internal/database/metadata_upstream_url_test.go @@ -0,0 +1,90 @@ +package database + +import ( + "database/sql" + "path/filepath" + "reflect" + "testing" + "time" +) + +func TestMigrationAddsMetadataUpstreamURL(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "existing.db") + sqlDB, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + if _, err := sqlDB.Exec(schemaSQLite); err != nil { + t.Fatalf("create schema: %v", err) + } + if _, err := sqlDB.Exec(`ALTER TABLE metadata_cache DROP COLUMN upstream_url`); err != nil { + t.Fatalf("drop upstream_url column: %v", err) + } + if _, err := sqlDB.Exec(`DELETE FROM migrations WHERE name = '023_add_metadata_upstream_url'`); err != nil { + t.Fatalf("forget migration: %v", err) + } + if err := sqlDB.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + db, err := Open(dbPath) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer func() { _ = db.Close() }() + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + has, err := db.HasColumn("metadata_cache", "upstream_url") + if err != nil || !has { + t.Errorf("metadata_cache.upstream_url after migration = %v, %v", has, err) + } +} + +func upsertMetadataURL(t *testing.T, db *DB, ecosystem, name, url string) { + t.Helper() + entry := &MetadataCacheEntry{Ecosystem: ecosystem, Name: name, StoragePath: "_metadata/" + name} + if url != "" { + entry.UpstreamURL = sql.NullString{String: url, Valid: true} + } + if err := db.UpsertMetadataCache(entry); err != nil { + t.Fatal(err) + } +} + +func TestUpsertMetadataCacheKeepsUpstreamURL(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertMetadataURL(t, db, "apk", "k1", "https://dl-cdn.alpinelinux.org/alpine/v3.20/main/aarch64/APKINDEX.tar.gz") + // A writer that doesn't know the URL leaves it as it was. + upsertMetadataURL(t, db, "apk", "k1", "") + got, err := db.GetMetadataCache("apk", "k1") + if err != nil || got == nil { + t.Fatalf("GetMetadataCache: %+v, %v", got, err) + } + if got.UpstreamURL.String != "https://dl-cdn.alpinelinux.org/alpine/v3.20/main/aarch64/APKINDEX.tar.gz" { + t.Errorf("upstream_url = %q", got.UpstreamURL.String) + } + }) +} + +func TestRecentMetadataDocuments(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertMetadataURL(t, db, "apk", "k1", "https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz") + upsertMetadataURL(t, db, "apk", "k2", "https://a.test/v3.19/main/aarch64/APKINDEX.tar.gz") + upsertMetadataURL(t, db, "apk", "k3", "") + upsertMetadataURL(t, db, "debian", "k4", "https://d.test/dists/bookworm/InRelease") + old := time.Now().Add(-30 * 24 * time.Hour).UTC() + if _, err := db.Exec(db.Rebind("UPDATE metadata_cache SET last_accessed_at = ?, updated_at = ? WHERE name = ?"), old, old, "k2"); err != nil { + t.Fatal(err) + } + + got, err := db.RecentMetadataDocuments("apk", time.Now().Add(-14*24*time.Hour)) + if err != nil { + t.Fatal(err) + } + want := []MetadataDocument{{Name: "k1", UpstreamURL: "https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz"}} + if !reflect.DeepEqual(got, want) { + t.Errorf("RecentMetadataDocuments = %+v, want %+v", got, want) + } + }) +} diff --git a/internal/database/most_used.go b/internal/database/most_used.go new file mode 100644 index 0000000..a365cb7 --- /dev/null +++ b/internal/database/most_used.go @@ -0,0 +1,80 @@ +package database + +import ( + "time" +) + +// UsedPackage is a package clients download from the cache, with the files +// cached for it. +type UsedPackage struct { + Ecosystem string + Name string + PURL string + // Hits counts the downloads of the files downloaded since the cutoff. + Hits int64 + Files []UsedFile +} + +// UsedFile is a cached file of a UsedPackage. +type UsedFile struct { + Version string + Filename string + UpstreamURL string +} + +// MostUsedPackages returns, for each ecosystem, the top packages with the most +// downloads among the cached files downloaded since cutoff, ordered by +// ecosystem and then by hits. Each lists every file cached for it. Files +// from local sources, which have no upstream, are left out, and so are +// files a mirror run fetched that no client has downloaded since. +func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) { + var rows []struct { + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + PURL string `db:"purl"` + Hits int64 `db:"hits"` + } + err := db.Select(&rows, db.Rebind(` + SELECT ecosystem, name, purl, hits FROM ( + SELECT p.ecosystem, p.name, p.purl, SUM(a.hit_count) AS hits, + ROW_NUMBER() OVER (PARTITION BY p.ecosystem ORDER BY SUM(a.hit_count) DESC, p.name) AS rank + FROM artifacts a + JOIN versions v ON v.purl = a.version_purl + JOIN packages p ON p.purl = v.package_purl + WHERE a.storage_path IS NOT NULL AND a.source = ? + AND (a.last_accessed_at >= ? OR (a.last_accessed_at IS NULL AND a.fetched_at >= ? + AND (a.mirrored_at IS NULL OR a.mirrored_at < a.fetched_at))) + GROUP BY p.ecosystem, p.name, p.purl + ) ranked + WHERE rank <= ? + ORDER BY ecosystem, hits DESC, name + `), SourceUpstream, cutoff.UTC(), cutoff.UTC(), top) + if err != nil { + return nil, err + } + packages := make([]UsedPackage, len(rows)) + for i, row := range rows { + packages[i] = UsedPackage{Ecosystem: row.Ecosystem, Name: row.Name, PURL: row.PURL, Hits: row.Hits} + var files []struct { + VersionPURL string `db:"version_purl"` + Filename string `db:"filename"` + UpstreamURL string `db:"upstream_url"` + } + err := db.Select(&files, db.Rebind(` + SELECT a.version_purl, a.filename, a.upstream_url + FROM artifacts a + JOIN versions v ON v.purl = a.version_purl + WHERE v.package_purl = ? AND a.storage_path IS NOT NULL AND a.source = ? + ORDER BY a.version_purl, a.filename + `), row.PURL, SourceUpstream) + if err != nil { + return nil, err + } + for _, f := range files { + packages[i].Files = append(packages[i].Files, UsedFile{ + Version: VersionFromPURL(f.VersionPURL), Filename: f.Filename, UpstreamURL: f.UpstreamURL, + }) + } + } + return packages, nil +} diff --git a/internal/database/most_used_test.go b/internal/database/most_used_test.go new file mode 100644 index 0000000..4b8693e --- /dev/null +++ b/internal/database/most_used_test.go @@ -0,0 +1,118 @@ +package database + +import ( + "reflect" + "testing" + "time" +) + +// seedUsedArtifact stores a cached artifact with hits, last downloaded at +// accessed. +func seedUsedArtifact(t *testing.T, db *DB, ecosystem, name, version, filename string, hits int64, accessed time.Time) { + t.Helper() + packagePURL := "pkg:" + ecosystem + "/" + name + versionPURL := packagePURL + "@" + version + if err := db.UpsertPackage(&Package{PURL: packagePURL, Ecosystem: ecosystem, Name: name}); err != nil { + t.Fatal(err) + } + if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: packagePURL}); err != nil { + t.Fatal(err) + } + if err := db.UpsertArtifact(&Artifact{VersionPURL: versionPURL, Filename: filename, UpstreamURL: "https://example.test/" + filename}); err != nil { + t.Fatal(err) + } + if err := db.MarkArtifactCached(versionPURL, filename, ecosystem+"/"+filename, "sha256:00", 1, "application/octet-stream"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET hit_count = ?, last_accessed_at = ? WHERE version_purl = ? AND filename = ?"), + hits, accessed.UTC(), versionPURL, filename); err != nil { + t.Fatal(err) + } +} + +func TestMostUsedPackages(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seedUsedArtifact(t, db, "npm", "lodash", "4.17.20", "lodash-4.17.20.tgz", 5, now.Add(-time.Hour)) + seedUsedArtifact(t, db, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", 4, now.Add(-20*24*time.Hour)) + seedUsedArtifact(t, db, "npm", "react", "18.3.1", "react-18.3.1.tgz", 10, now.Add(-time.Hour)) + seedUsedArtifact(t, db, "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", 100, now.Add(-30*24*time.Hour)) + seedUsedArtifact(t, db, "cargo", "serde", "1.0.200", "serde-1.0.200.crate", 1, now.Add(-time.Hour)) + + // Not cached: never counts. + if err := db.UpsertArtifact(&Artifact{VersionPURL: "pkg:npm/react@19.0.0", Filename: "react-19.0.0.tgz", UpstreamURL: "https://example.test/x"}); err != nil { + t.Fatal(err) + } + + got, err := db.MostUsedPackages(now.Add(-14*24*time.Hour), 2) + if err != nil { + t.Fatal(err) + } + want := []UsedPackage{ + {Ecosystem: "cargo", Name: "serde", PURL: "pkg:cargo/serde", Hits: 1, Files: []UsedFile{ + {Version: "1.0.200", Filename: "serde-1.0.200.crate", UpstreamURL: "https://example.test/serde-1.0.200.crate"}, + }}, + // react has more hits than lodash. Every cached file of lodash is + // listed, but only recent downloads count. + {Ecosystem: "npm", Name: "react", PURL: "pkg:npm/react", Hits: 10, Files: []UsedFile{ + {Version: "18.3.1", Filename: "react-18.3.1.tgz", UpstreamURL: "https://example.test/react-18.3.1.tgz"}, + }}, + {Ecosystem: "npm", Name: "lodash", PURL: "pkg:npm/lodash", Hits: 5, Files: []UsedFile{ + {Version: "4.17.20", Filename: "lodash-4.17.20.tgz", UpstreamURL: "https://example.test/lodash-4.17.20.tgz"}, + {Version: "4.17.21", Filename: "lodash-4.17.21.tgz", UpstreamURL: "https://example.test/lodash-4.17.21.tgz"}, + }}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("MostUsedPackages =\n%+v\nwant\n%+v", got, want) + } + + top1, err := db.MostUsedPackages(now.Add(-14*24*time.Hour), 1) + if err != nil { + t.Fatal(err) + } + if len(top1) != 2 || top1[0].Name != "serde" || top1[1].Name != "react" { + t.Errorf("top 1 per ecosystem = %+v, want serde and react", top1) + } + }) +} + +func TestMostUsedPackagesSkipsLocalArtifacts(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seedUsedArtifact(t, db, "npm", "internal-lib", "1.0.0", "internal-lib-1.0.0.tgz", 50, now) + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET source = ? WHERE filename = ?"), SourceLocal, "internal-lib-1.0.0.tgz"); err != nil { + t.Fatal(err) + } + got, err := db.MostUsedPackages(now.Add(-time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Errorf("MostUsedPackages = %+v, want none: local packages have no upstream to refresh from", got) + } + }) +} + +func TestMostUsedPackagesSkipsMirroredArtifactsNotDownloaded(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seedUsedArtifact(t, db, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", 0, now) + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET last_accessed_at = NULL, mirrored_at = ?"), time.Now().UTC()); err != nil { + t.Fatal(err) + } + got, err := db.MostUsedPackages(now.Add(-time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Errorf("MostUsedPackages = %+v, want none: a mirror fetched it and no client has since", got) + } + + if err := db.RecordArtifactHit("pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz"); err != nil { + t.Fatal(err) + } + if got, err = db.MostUsedPackages(now.Add(-time.Hour), 10); err != nil || len(got) != 1 { + t.Errorf("MostUsedPackages = %+v, %v; want lodash once a client downloaded it", got, err) + } + }) +} diff --git a/internal/database/projects.go b/internal/database/projects.go new file mode 100644 index 0000000..c14b0d6 --- /dev/null +++ b/internal/database/projects.go @@ -0,0 +1,222 @@ +package database + +import ( + "database/sql" + "errors" + "fmt" + "time" +) + +// Project is a repository that uploaded an SBOM to the project inventory. +type Project struct { + ID int64 `db:"id"` + Name string `db:"name"` + Ref sql.NullString `db:"ref"` + Branch sql.NullString `db:"branch"` + UploadedAt time.Time `db:"uploaded_at"` + // Components is the number of package versions recorded for it. + Components int64 `db:"components"` +} + +// ProjectComponent is a package version a project uses. PackagePURL is the +// unversioned package URL, as packages.purl records it. +type ProjectComponent struct { + PackagePURL string `db:"package_purl"` + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + Version string `db:"version"` +} + +// ProjectComponentInfo is a project's component together with the package +// record the proxy holds for it, if any. +type ProjectComponentInfo struct { + ProjectComponent + CachedEcosystem sql.NullString `db:"cached_ecosystem"` + CachedName sql.NullString `db:"cached_name"` + LatestVersion sql.NullString `db:"latest_version"` +} + +// ProjectUsage is a project using a version of a package. +type ProjectUsage struct { + Name string `db:"name"` + Ref sql.NullString `db:"ref"` + Branch sql.NullString `db:"branch"` + UploadedAt time.Time `db:"uploaded_at"` + Version string `db:"version"` +} + +const projectColumns = ` + SELECT pr.id, pr.name, pr.ref, pr.branch, pr.uploaded_at, + (SELECT COUNT(*) FROM project_components pc WHERE pc.project_id = pr.id) AS components` + +// SaveProject records the components of a project, replacing everything +// recorded for it before, in one transaction. Readers see either the old or +// the new set, never a mix. +func (db *DB) SaveProject(name, ref, branch string, uploadedAt time.Time, components []ProjectComponent) error { + tx, err := db.Beginx() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + // The upsert locks the project row, so concurrent uploads of the same + // project and pruning take turns. + if _, err := tx.Exec(db.Rebind(` + INSERT INTO projects (name, ref, branch, uploaded_at) VALUES (?, ?, ?, ?) + ON CONFLICT(name) DO UPDATE SET ref = excluded.ref, branch = excluded.branch, uploaded_at = excluded.uploaded_at + `), name, nullString(ref), nullString(branch), uploadedAt.UTC()); err != nil { + return fmt.Errorf("recording project: %w", err) + } + var id int64 + if err := tx.Get(&id, db.Rebind(`SELECT id FROM projects WHERE name = ?`), name); err != nil { + return fmt.Errorf("reading project id: %w", err) + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return fmt.Errorf("removing old components: %w", err) + } + + insert, err := tx.Preparex(db.Rebind(` + INSERT INTO project_components (project_id, package_purl, ecosystem, name, version) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(project_id, package_purl, version) DO NOTHING + `)) + if err != nil { + return err + } + defer func() { _ = insert.Close() }() + for _, c := range components { + if _, err := insert.Exec(id, c.PackagePURL, c.Ecosystem, c.Name, c.Version); err != nil { + return fmt.Errorf("recording component %s@%s: %w", c.PackagePURL, c.Version, err) + } + } + return tx.Commit() +} + +// DeleteProject removes a project and its components. It reports whether the +// project existed. +func (db *DB) DeleteProject(name string) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + var id int64 + err = tx.Get(&id, db.Rebind(`DELETE FROM projects WHERE name = ? RETURNING id`), name) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("deleting project: %w", err) + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return false, fmt.Errorf("deleting project components: %w", err) + } + return true, tx.Commit() +} + +// PruneProjects removes the projects last uploaded before cutoff, with their +// components, and returns how many it removed. A project uploaded again while +// the prune runs is kept. +func (db *DB) PruneProjects(cutoff time.Time) (int, error) { + var ids []int64 + if err := db.Select(&ids, db.Rebind(`SELECT id FROM projects WHERE uploaded_at < ?`), cutoff.UTC()); err != nil { + return 0, fmt.Errorf("listing expired projects: %w", err) + } + pruned := 0 + for _, id := range ids { + ok, err := db.pruneProject(id, cutoff) + if err != nil { + return pruned, err + } + if ok { + pruned++ + } + } + return pruned, nil +} + +func (db *DB) pruneProject(id int64, cutoff time.Time) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + res, err := tx.Exec(db.Rebind(`DELETE FROM projects WHERE id = ? AND uploaded_at < ?`), id, cutoff.UTC()) + if err != nil { + return false, fmt.Errorf("pruning project: %w", err) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return false, err + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return false, fmt.Errorf("pruning project components: %w", err) + } + return true, tx.Commit() +} + +// ListProjects returns all projects ordered by name. +func (db *DB) ListProjects() ([]Project, error) { + projects := []Project{} + if err := db.Select(&projects, projectColumns+` FROM projects pr ORDER BY pr.name`); err != nil { + return nil, err + } + return projects, nil +} + +// GetProject returns the named project, or nil if there is none. +func (db *DB) GetProject(name string) (*Project, error) { + var p Project + err := db.Get(&p, db.Rebind(projectColumns+` FROM projects pr WHERE pr.name = ?`), name) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &p, nil +} + +// GetProjectComponents returns the components of a project, ordered by +// ecosystem, name and version, with the package record the proxy holds for +// each. +func (db *DB) GetProjectComponents(projectID int64) ([]ProjectComponentInfo, error) { + components := []ProjectComponentInfo{} + err := db.Select(&components, db.Rebind(` + SELECT c.package_purl, c.ecosystem, c.name, c.version, + p.ecosystem AS cached_ecosystem, p.name AS cached_name, p.latest_version + FROM project_components c + LEFT JOIN packages p ON p.purl = c.package_purl + WHERE c.project_id = ? + ORDER BY c.ecosystem, c.name, c.version + `), projectID) + if err != nil { + return nil, err + } + return components, nil +} + +// GetProjectsUsingPackage returns the projects using a package, or only +// those using version when it is not empty, ordered by project and version. +func (db *DB) GetProjectsUsingPackage(packagePURL, version string) ([]ProjectUsage, error) { + query := ` + SELECT pr.name, pr.ref, pr.branch, pr.uploaded_at, c.version + FROM project_components c + JOIN projects pr ON pr.id = c.project_id + WHERE c.package_purl = ?` + args := []any{packagePURL} + if version != "" { + query += ` AND c.version = ?` + args = append(args, version) + } + usages := []ProjectUsage{} + if err := db.Select(&usages, db.Rebind(query+` ORDER BY pr.name, c.version`), args...); err != nil { + return nil, err + } + return usages, nil +} + +func nullString(s string) sql.NullString { + return sql.NullString{String: s, Valid: s != ""} +} diff --git a/internal/database/projects_test.go b/internal/database/projects_test.go new file mode 100644 index 0000000..27bb34c --- /dev/null +++ b/internal/database/projects_test.go @@ -0,0 +1,259 @@ +package database + +import ( + "database/sql" + "slices" + "sync" + "testing" + "time" +) + +var ( + lodash20 = ProjectComponent{PackagePURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", Version: "4.17.20"} + lodash21 = ProjectComponent{PackagePURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", Version: "4.17.21"} + console = ProjectComponent{PackagePURL: "pkg:composer/symfony/console", Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1"} +) + +func saveProject(t *testing.T, db *DB, name string, at time.Time, components ...ProjectComponent) { + t.Helper() + if err := db.SaveProject(name, "", "", at, components); err != nil { + t.Fatalf("SaveProject(%s): %v", name, err) + } +} + +func usageNames(usages []ProjectUsage) []string { + var names []string + for _, u := range usages { + names = append(names, u.Name+"@"+u.Version) + } + return names +} + +func TestSaveProjectReplacesComponents(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now().UTC().Truncate(time.Second) + if err := db.SaveProject("acme/web", "abc123", "main", now, []ProjectComponent{lodash20, console, lodash20}); err != nil { + t.Fatal(err) + } + saveProject(t, db, "acme/web", now.Add(time.Minute), lodash21) + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + if project.Components != 1 || project.Ref.Valid || project.Branch.Valid { + t.Errorf("project = %+v, want one component and ref and branch cleared", project) + } + if !project.UploadedAt.Equal(now.Add(time.Minute)) { + t.Errorf("uploaded_at = %v, want %v", project.UploadedAt, now.Add(time.Minute)) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].ProjectComponent != lodash21 { + t.Errorf("components = %+v, want only lodash 4.17.21", components) + } + + missing, err := db.GetProject("acme/other") + if err != nil || missing != nil { + t.Errorf("GetProject(missing) = %v, %v", missing, err) + } + }) +} + +func TestGetProjectsUsingPackage(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/web", now, lodash20, console) + saveProject(t, db, "acme/api", now, lodash21) + saveProject(t, db, "acme/cli", now, console) + + all, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(all), []string{"acme/api@4.17.21", "acme/web@4.17.20"}; !slices.Equal(got, want) { + t.Errorf("any version = %v, want %v", got, want) + } + one, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "4.17.20") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(one), []string{"acme/web@4.17.20"}; !slices.Equal(got, want) { + t.Errorf("4.17.20 = %v, want %v", got, want) + } + none, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "1.0.0") + if err != nil || len(none) != 0 { + t.Errorf("unused version = %v, %v", none, err) + } + }) +} + +func TestListAndDeleteProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/web", now, lodash20, console) + saveProject(t, db, "acme/api", now) + + projects, err := db.ListProjects() + if err != nil { + t.Fatal(err) + } + if len(projects) != 2 || projects[0].Name != "acme/api" || projects[0].Components != 0 || projects[1].Components != 2 { + t.Fatalf("projects = %+v", projects) + } + + deleted, err := db.DeleteProject("acme/web") + if err != nil || !deleted { + t.Fatalf("DeleteProject = %v, %v", deleted, err) + } + deleted, err = db.DeleteProject("acme/web") + if err != nil || deleted { + t.Errorf("second DeleteProject = %v, %v, want false", deleted, err) + } + var rows int + if err := db.Get(&rows, `SELECT COUNT(*) FROM project_components`); err != nil { + t.Fatal(err) + } + if rows != 0 { + t.Errorf("%d component rows left after the delete", rows) + } + }) +} + +func TestPruneProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/old", now.Add(-100*24*time.Hour), lodash20) + saveProject(t, db, "acme/new", now.Add(-time.Hour), lodash21) + + pruned, err := db.PruneProjects(now.Add(-90 * 24 * time.Hour)) + if err != nil { + t.Fatal(err) + } + if pruned != 1 { + t.Errorf("pruned = %d, want 1", pruned) + } + projects, err := db.ListProjects() + if err != nil { + t.Fatal(err) + } + if len(projects) != 1 || projects[0].Name != "acme/new" { + t.Errorf("projects after prune = %+v", projects) + } + usages, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(usages), []string{"acme/new@4.17.21"}; !slices.Equal(got, want) { + t.Errorf("usages after prune = %v, want %v", got, want) + } + }) +} + +func TestProjectComponentsJoinPackages(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + if err := db.UpsertPackage(&Package{ + PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", + LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } + saveProject(t, db, "acme/web", time.Now(), lodash20, console) + saveProject(t, db, "acme/api", time.Now(), lodash21) + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) != 2 { + t.Fatalf("components = %+v", components) + } + // Ordered by ecosystem: composer first, which the proxy has no record of. + if components[0].CachedName.Valid { + t.Errorf("uncached component has a package record: %+v", components[0]) + } + if components[1].CachedName.String != "lodash" || components[1].LatestVersion.String != "4.17.21" { + t.Errorf("cached component = %+v", components[1]) + } + + }) +} + +func TestMigrateAddsProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + // A database from before the project inventory. + for _, table := range []string{"project_components", "projects"} { + if _, err := db.Exec("DROP TABLE " + table); err != nil { + t.Fatalf("dropping %s: %v", table, err) + } + } + if _, err := db.Exec(db.Rebind("DELETE FROM migrations WHERE name = ?"), "021_add_projects"); err != nil { + t.Fatalf("forgetting the migration: %v", err) + } + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + saveProject(t, db, "acme/web", time.Now(), lodash20) + usages, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "4.17.20") + if err != nil || len(usages) != 1 { + t.Fatalf("GetProjectsUsingPackage after migration = %v, %v", usages, err) + } + }) +} + +// TestSaveProjectConcurrentUploads checks that concurrent uploads of one +// project all succeed and leave the components of exactly one of them. +func TestSaveProjectConcurrentUploads(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const uploads = 8 + sets := make([][]ProjectComponent, uploads) + for i := range sets { + for j := 0; j <= i; j++ { + sets[i] = append(sets[i], ProjectComponent{ + PackagePURL: "pkg:npm/dep" + string(rune('a'+j)), Ecosystem: "npm", + Name: "dep" + string(rune('a'+j)), Version: "1.0." + string(rune('0'+i)), + }) + } + } + now := time.Now().UTC() + errs := make(chan error, uploads) + var wg sync.WaitGroup + for i := range uploads { + wg.Go(func() { errs <- db.SaveProject("acme/web", "", "", now, sets[i]) }) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Errorf("SaveProject: %v", err) + } + } + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) == 0 || len(components) > uploads { + t.Fatalf("%d components, want those of one upload", len(components)) + } + version := components[0].Version + for _, c := range components { + if c.Version != version { + t.Fatalf("components mix uploads: %+v", components) + } + } + if want := int(version[len(version)-1]-'0') + 1; len(components) != want { + t.Errorf("%d components of the upload with version %s, want %d", len(components), version, want) + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e8..6f69317 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -4,7 +4,9 @@ import ( "database/sql" "errors" "fmt" + "slices" "sort" + "strings" "time" "github.com/git-pkgs/artifacts" @@ -183,7 +185,7 @@ func (db *DB) upsertVersion(exec execer, v *Version) error { _, err := exec.Exec(query, v.PURL, v.PackagePURL, v.License, v.Integrity, - v.PublishedAt, v.Yanked, v.EnrichedAt, now, now, + inUTC(v.PublishedAt), v.Yanked, v.EnrichedAt, now, now, ) if err != nil { return fmt.Errorf("upserting version: %w", err) @@ -216,13 +218,32 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt ` } - _, err := db.Exec(query, versionPURL, packagePURL, publishedAt, now, now) + _, err := db.Exec(query, versionPURL, packagePURL, publishedAt.UTC(), now, now) if err != nil { return fmt.Errorf("setting version publish time: %w", err) } return nil } +// GetFetchedVersionPURLs returns the versions of a package that have an +// artifact row, cached now or evicted since. Versions without one were only +// seen in metadata. +func (db *DB) GetFetchedVersionPURLs(packagePURL string) (map[string]bool, error) { + var purls []string + query := db.Rebind(` + SELECT DISTINCT version_purl FROM artifacts + WHERE version_purl IN (SELECT purl FROM versions WHERE package_purl = ?) + `) + if err := db.Select(&purls, query, packagePURL); err != nil { + return nil, fmt.Errorf("listing fetched versions: %w", err) + } + fetched := make(map[string]bool, len(purls)) + for _, p := range purls { + fetched[p] = true + } + return fetched, nil +} + // Artifact queries func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { @@ -645,6 +666,7 @@ func (db *DB) GetDuePendingDeletes(cutoff time.Time, limit int) ([]string, error WHERE queued_at < ? AND NOT EXISTS (SELECT 1 FROM artifacts WHERE artifacts.storage_path = pending_deletes.path) AND NOT EXISTS (SELECT 1 FROM metadata_cache WHERE metadata_cache.storage_path = pending_deletes.path) + AND NOT EXISTS (SELECT 1 FROM metadata_snapshots WHERE metadata_snapshots.storage_path = pending_deletes.path) ORDER BY queued_at LIMIT ? `) @@ -827,7 +849,7 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i } var results []SearchResult - searchPattern := "%" + query + "%" + searchPattern := nameSearchPattern(query) var sqlQuery string var args []any @@ -842,7 +864,7 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND ` + condition + ` AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND ` + condition + ` AND a.storage_path IS NOT NULL GROUP BY p.purl, p.ecosystem, p.name, p.latest_version, p.license ORDER BY hits DESC LIMIT ? OFFSET ? @@ -857,7 +879,7 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND a.storage_path IS NOT NULL GROUP BY p.purl, p.ecosystem, p.name, p.latest_version, p.license ORDER BY hits DESC LIMIT ? OFFSET ? @@ -872,6 +894,18 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i return results, nil } +// nameSearchCondition matches package names containing a search pattern +// from nameSearchPattern, ignoring case on SQLite and Postgres alike. +const nameSearchCondition = `LOWER(p.name) LIKE LOWER(?) ESCAPE '!'` + +// nameSearchPattern is the LIKE pattern for names containing query. A % +// in query is matched literally, but _ stays a single-character wildcard +// so typing_extensions still finds typing-extensions. +func nameSearchPattern(query string) string { + escaped := strings.NewReplacer("!", "!!", "%", "!%").Replace(query) + return "%" + escaped + "%" +} + func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) { hasArtifacts, err := db.HasTable("artifacts") if err != nil { @@ -881,7 +915,7 @@ func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) return 0, nil } - searchPattern := "%" + query + "%" + searchPattern := nameSearchPattern(query) var sqlQuery string var args []any @@ -892,7 +926,7 @@ func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND ` + condition + ` AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND ` + condition + ` AND a.storage_path IS NOT NULL `) args = append([]any{searchPattern}, ecosystemArgs...) } else { @@ -901,7 +935,7 @@ func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND a.storage_path IS NOT NULL `) args = []any{searchPattern} } @@ -1299,7 +1333,7 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err var entry MetadataCacheEntry query := db.Rebind(` SELECT id, ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at + content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at FROM metadata_cache WHERE ecosystem = ? AND name = ? `) err := db.Get(&entry, query, ecosystem, name) @@ -1312,6 +1346,26 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err return &entry, nil } +// MetadataDocument names a cached metadata document and where it was +// fetched from. +type MetadataDocument struct { + Name string `db:"name"` + UpstreamURL string `db:"upstream_url"` +} + +// RecentMetadataDocuments returns the documents of an ecosystem cached or +// served since cutoff whose upstream URL was recorded. +func (db *DB) RecentMetadataDocuments(ecosystem string, cutoff time.Time) ([]MetadataDocument, error) { + var docs []MetadataDocument + err := db.Select(&docs, db.Rebind(` + SELECT name, upstream_url FROM metadata_cache + WHERE ecosystem = ? AND upstream_url IS NOT NULL + AND COALESCE(last_accessed_at, updated_at) >= ? + ORDER BY upstream_url, name + `), ecosystem, cutoff.UTC()) + return docs, err +} + // RefreshMetadataCache sets fetched_at on the metadata cache row for ecosystem // and name, provided the row still records digest, and reports whether it did. // It is for an upstream 304, which leaves the stored document as it is. @@ -1328,6 +1382,14 @@ func (db *DB) RefreshMetadataCache(ecosystem, name, digest string, fetchedAt tim return n > 0, err } +// SetMetadataUpstreamURL records the upstream URL of the metadata cache row +// for ecosystem and name, for rows cached before URLs were recorded. +func (db *DB) SetMetadataUpstreamURL(ecosystem, name, upstreamURL string) error { + _, err := db.Exec(db.Rebind(`UPDATE metadata_cache SET upstream_url = ? WHERE ecosystem = ? AND name = ?`), + upstreamURL, ecosystem, name) + return err +} + // MoveMetadataCache points the metadata cache row for ecosystem and name at // to, provided it is still at from and records digest, and reports whether // it did. The caller has copied the document from from to to. @@ -1357,6 +1419,50 @@ func (db *DB) ForgetMetadataDocument(ecosystem, name, from, to string) error { return err } +// expireMetadataBatch is how many names one ExpireMetadataCache statement +// covers, well below the bind parameter limits of SQLite and Postgres. +const expireMetadataBatch = 500 + +// ExpireMetadataCache clears fetched_at on the metadata cache rows for +// ecosystem and names that were fetched before cutoff, and returns the names +// of the rows it changed. The rows keep their documents and validators, so +// the next lookup of each revalidates it upstream instead of serving it as +// fresh. It is for an upstream that reports which documents changed. +func (db *DB) ExpireMetadataCache(ecosystem string, names []string, cutoff time.Time) ([]string, error) { + var expired []string + for batch := range slices.Chunk(names, expireMetadataBatch) { + args := []any{time.Now(), ecosystem, cutoff.UTC()} + for _, name := range batch { + args = append(args, name) + } + var got []string + err := db.DB.Select(&got, db.Rebind(` + UPDATE metadata_cache SET fetched_at = NULL, updated_at = ? + WHERE ecosystem = ? AND fetched_at < ? + AND name IN (`+strings.TrimSuffix(strings.Repeat("?, ", len(batch)), ", ")+`) + RETURNING name + `), args...) + if err != nil { + return expired, err + } + expired = append(expired, got...) + } + return expired, nil +} + +// ExpireAllMetadataCache is ExpireMetadataCache for every row of ecosystem, +// except those of packages routed to another registry (named "routes/..."), +// for an upstream that can't say which of its documents changed. +func (db *DB) ExpireAllMetadataCache(ecosystem string, cutoff time.Time) ([]string, error) { + var expired []string + err := db.DB.Select(&expired, db.Rebind(` + UPDATE metadata_cache SET fetched_at = NULL, updated_at = ? + WHERE ecosystem = ? AND fetched_at < ? AND name NOT LIKE 'routes/%' + RETURNING name + `), time.Now(), ecosystem, cutoff.UTC()) + return expired, err +} + // UpsertMetadataCache writes entry and marks it accessed now, since a // document is stored because a request asked for it. func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { @@ -1373,8 +1479,8 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { if db.dialect == DialectPostgres { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = EXCLUDED.storage_path, etag = EXCLUDED.etag, @@ -1382,6 +1488,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { content_type = EXCLUDED.content_type, content_encoding = EXCLUDED.content_encoding, content_digest = EXCLUDED.content_digest, + upstream_url = COALESCE(EXCLUDED.upstream_url, metadata_cache.upstream_url), size = EXCLUDED.size, last_modified = EXCLUDED.last_modified, fetched_at = EXCLUDED.fetched_at, @@ -1391,8 +1498,8 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { } else { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = excluded.storage_path, etag = excluded.etag, @@ -1400,6 +1507,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { content_type = excluded.content_type, content_encoding = excluded.content_encoding, content_digest = excluded.content_digest, + upstream_url = COALESCE(excluded.upstream_url, metadata_cache.upstream_url), size = excluded.size, last_modified = excluded.last_modified, fetched_at = excluded.fetched_at, @@ -1410,7 +1518,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { _, err := db.Exec(query, entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag, entry.Link, - entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.Size, entry.LastModified, fetchedAt, now.UTC(), now, now, + entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.UpstreamURL, entry.Size, entry.LastModified, fetchedAt, now.UTC(), now, now, ) if err != nil { return fmt.Errorf("upserting metadata cache: %w", err) @@ -1458,15 +1566,18 @@ func (db *DB) GetEcosystemStats() ([]EcosystemStats, error) { return nil, err } - // Left joined and bucketed for the same reason as artifacts below: a version - // whose package row is missing would otherwise vanish here while still - // counting in GetCacheStats' COUNT(*), leaving the two unable to reconcile. + // Left joined for the same reason as artifacts below: a version whose + // package row is missing would otherwise vanish here while still counting + // in GetCacheStats' COUNT(*), leaving the two unable to reconcile. Such + // versions are mostly publish times cooldown stored for versions nobody + // downloaded, so they count under the type of their package PURL, and + // only a PURL that can't be parsed is unattributed. if err := db.eachCount(` - SELECT COALESCE(p.ecosystem, '`+unattributedEcosystem+`'), COUNT(*) + SELECT COALESCE(p.ecosystem, '`+orphanVersionPrefix+`' || v.package_purl), COUNT(*) FROM versions v LEFT JOIN packages p ON p.purl = v.package_purl - GROUP BY COALESCE(p.ecosystem, '`+unattributedEcosystem+`') - `, func(ecosystem string, n int64) { get(ecosystem).Versions = n }); err != nil { + GROUP BY COALESCE(p.ecosystem, '`+orphanVersionPrefix+`' || v.package_purl) + `, func(key string, n int64) { get(versionEcosystem(key)).Versions += n }); err != nil { return nil, err } @@ -1549,6 +1660,26 @@ func mergeAliasedEcosystems(stats []EcosystemStats) []EcosystemStats { // Grafana. Bucketing them keeps the two reconcilable. const unattributedEcosystem = "unattributed" +// orphanVersionPrefix marks the package PURL of a version without a package +// row in GetEcosystemStats. No ecosystem name starts with it. +const orphanVersionPrefix = "~" + +// versionEcosystem returns the ecosystem a GetEcosystemStats version count +// is for: the package row's, or the type of the package PURL of a version +// without one. The ecosystems that store publish times are named like their +// PURL types. +func versionEcosystem(key string) string { + packagePURL, orphan := strings.CutPrefix(key, orphanVersionPrefix) + if !orphan { + return key + } + p, err := purl.Parse(packagePURL) + if err != nil { + return unattributedEcosystem + } + return p.Type +} + func (db *DB) eachArtifactStat(get func(string) *EcosystemStats) error { rows, err := db.Query(` SELECT COALESCE(p.ecosystem, '` + unattributedEcosystem + `'), diff --git a/internal/database/queries_packages_list_test.go b/internal/database/queries_packages_list_test.go index 325fee7..5eaa402 100644 --- a/internal/database/queries_packages_list_test.go +++ b/internal/database/queries_packages_list_test.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "maps" + "slices" "testing" "time" ) @@ -250,3 +251,44 @@ func TestCachedPackageFiltersMatchEcosystemAliases(t *testing.T) { } }) } + +func TestSearchPackagesIgnoresCaseAndEscapesPercent(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedArtifact(t, db, "pypi", "Django", "5.0.0", 100, 1) + seedArtifact(t, db, "pypi", "typing-extensions", "4.12.0", 100, 1) + seedArtifact(t, db, "npm", "my_pkg", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", "100%-done", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", "1000-done", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", `back\slash`, "1.0.0", 100, 1) + + for _, tc := range []struct { + query string + want []string + }{ + {"django", []string{"Django"}}, + {"DJANGO", []string{"Django"}}, + {"my_pkg", []string{"my_pkg"}}, + // _ stays a single-character wildcard, so it matches - too. + {"typing_extensions", []string{"typing-extensions"}}, + {"typing-extensions", []string{"typing-extensions"}}, + {"100%", []string{"100%-done"}}, + {`k\s`, []string{`back\slash`}}, + } { + found, err := db.SearchPackages(tc.query, "", 50, 0) + if err != nil { + t.Fatalf("SearchPackages(%q): %v", tc.query, err) + } + var names []string + for _, p := range found { + names = append(names, p.Name) + } + if !slices.Equal(names, tc.want) { + t.Errorf("SearchPackages(%q) = %v, want %v", tc.query, names, tc.want) + } + count, err := db.CountSearchResults(tc.query, "") + if err != nil || count != int64(len(tc.want)) { + t.Errorf("CountSearchResults(%q) = %d, %v, want %d", tc.query, count, err, len(tc.want)) + } + } + }) +} diff --git a/internal/database/retention.go b/internal/database/retention.go index 90fffd2..a2a59e0 100644 --- a/internal/database/retention.go +++ b/internal/database/retention.go @@ -17,28 +17,33 @@ type RetentionCandidate struct { Size sql.NullInt64 `db:"size"` FetchedAt sql.NullTime `db:"fetched_at"` LastAccessedAt sql.NullTime `db:"last_accessed_at"` + MirroredAt sql.NullTime `db:"mirrored_at"` Ecosystem string `db:"ecosystem"` PackagePURL string `db:"package_purl"` } -// ExpiredBefore reports whether the artifact was neither fetched nor -// accessed at or after cutoff. Both are checked: clearing a record keeps its -// last access time, so a record fetched again after an eviction has a fresh -// fetch time next to an old access time. +// ExpiredBefore reports whether the artifact was neither fetched, accessed +// nor found by a mirror run at or after cutoff. Fetch and access are both +// checked: clearing a record keeps its last access time, so a record fetched +// again after an eviction has a fresh fetch time next to an old access time. func (c RetentionCandidate) ExpiredBefore(cutoff time.Time) bool { if !c.FetchedAt.Valid || !c.FetchedAt.Time.Before(cutoff) { return false } + if c.MirroredAt.Valid && !c.MirroredAt.Time.Before(cutoff) { + return false + } return !c.LastAccessedAt.Valid || c.LastAccessedAt.Time.Before(cutoff) } -// expiredCondition selects cached artifacts fetched before the cutoff and not -// accessed since. It takes the cutoff twice. +// expiredCondition selects cached artifacts fetched before the cutoff and +// neither accessed nor mirrored since. It takes the cutoff three times. const expiredCondition = ` storage_path IS NOT NULL AND source <> 'local' AND fetched_at < ? - AND (last_accessed_at IS NULL OR last_accessed_at < ?)` + AND (last_accessed_at IS NULL OR last_accessed_at < ?) + AND (mirrored_at IS NULL OR mirrored_at < ?)` // MaxArtifactID returns the highest artifact id, or zero for an empty table. func (db *DB) MaxArtifactID() (int64, error) { @@ -56,7 +61,7 @@ func (db *DB) GetRetentionCandidates(fromID, toID int64, cutoff time.Time) ([]Re var candidates []RetentionCandidate query := db.Rebind(` SELECT a.id, a.version_purl, a.filename, a.storage_path, a.size, - a.fetched_at, a.last_accessed_at, p.ecosystem, p.purl AS package_purl + a.fetched_at, a.last_accessed_at, a.mirrored_at, p.ecosystem, p.purl AS package_purl FROM ( SELECT * FROM artifacts WHERE id > ? AND id <= ? AND` + expiredCondition + ` @@ -65,12 +70,28 @@ func (db *DB) GetRetentionCandidates(fromID, toID int64, cutoff time.Time) ([]Re JOIN packages p ON p.purl = v.package_purl ORDER BY a.id `) - if err := db.Select(&candidates, query, fromID, toID, cutoff.UTC(), cutoff.UTC()); err != nil { + if err := db.Select(&candidates, query, fromID, toID, cutoff.UTC(), cutoff.UTC(), cutoff.UTC()); err != nil { return nil, err } return candidates, nil } +// CachedVersionPURLs returns the versions of a package that have at least one +// file cached from upstream. +func (db *DB) CachedVersionPURLs(packagePURL string) ([]string, error) { + var purls []string + query := db.Rebind(` + SELECT DISTINCT a.version_purl + FROM versions v + JOIN artifacts a ON a.version_purl = v.purl + WHERE v.package_purl = ? AND a.storage_path IS NOT NULL AND a.source <> 'local' + `) + if err := db.Select(&purls, query, packagePURL); err != nil { + return nil, err + } + return purls, nil +} + // ClearExpiredArtifact marks an artifact uncached like ClearArtifactCache, // but only while it is still expired before cutoff, so an artifact served // between the scan and the clear stays cached. It queues storagePath for @@ -89,7 +110,7 @@ func (db *DB) ClearExpiredArtifact(versionPURL, filename, storagePath string, cu SET storage_path = NULL, content_hash = NULL, size = NULL, content_type = NULL, fetched_at = NULL, updated_at = ? WHERE version_purl = ? AND filename = ? AND storage_path = ? AND` + expiredCondition) - res, err := tx.Exec(query, time.Now(), versionPURL, filename, storagePath, cutoff.UTC(), cutoff.UTC()) + res, err := tx.Exec(query, time.Now(), versionPURL, filename, storagePath, cutoff.UTC(), cutoff.UTC(), cutoff.UTC()) if err != nil { return false, err } @@ -103,6 +124,16 @@ func (db *DB) ClearExpiredArtifact(versionPURL, filename, storagePath string, cu return true, tx.Commit() } +// MarkArtifactMirrored records that a mirror run found the artifact cached, +// which keeps it from retention without counting as a cache hit. +func (db *DB) MarkArtifactMirrored(versionPURL, filename string) error { + _, err := db.Exec(db.Rebind(` + UPDATE artifacts SET mirrored_at = ? + WHERE version_purl = ? AND filename = ? AND storage_path IS NOT NULL + `), time.Now().UTC(), versionPURL, filename) + return err +} + // GetArtifactEcosystem returns the packages.ecosystem value of the package an // artifact version belongs to, or "" when it has no package record. func (db *DB) GetArtifactEcosystem(versionPURL string) (string, error) { diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go index bd9ad28..62582ec 100644 --- a/internal/database/retention_test.go +++ b/internal/database/retention_test.go @@ -341,18 +341,20 @@ func TestRetentionCandidateExpiredBefore(t *testing.T) { recent := sql.NullTime{Time: now, Valid: true} tests := []struct { - name string - fetched, accessed sql.NullTime - want bool + name string + fetched, accessed, mirrored sql.NullTime + want bool }{ - {"fetched and accessed before", old, old, true}, - {"never served", old, sql.NullTime{}, true}, - {"accessed since", old, recent, false}, - {"fetched again since", recent, old, false}, - {"not cached", sql.NullTime{}, old, false}, + {"fetched and accessed before", old, old, sql.NullTime{}, true}, + {"never served", old, sql.NullTime{}, sql.NullTime{}, true}, + {"accessed since", old, recent, sql.NullTime{}, false}, + {"fetched again since", recent, old, sql.NullTime{}, false}, + {"not cached", sql.NullTime{}, old, sql.NullTime{}, false}, + {"mirrored since", old, old, recent, false}, + {"mirrored before", old, old, old, true}, } for _, tt := range tests { - c := RetentionCandidate{FetchedAt: tt.fetched, LastAccessedAt: tt.accessed} + c := RetentionCandidate{FetchedAt: tt.fetched, LastAccessedAt: tt.accessed, MirroredAt: tt.mirrored} if got := c.ExpiredBefore(cutoff); got != tt.want { t.Errorf("%s: ExpiredBefore() = %v, want %v", tt.name, got, tt.want) } @@ -427,3 +429,46 @@ func TestArtifactFilenameIndex(t *testing.T) { } }) } + +// CachedVersionPURLs lists each version of a package once while it has a +// file cached from upstream, and leaves out other packages, cleared files +// and pushed artifacts. +func TestCachedVersionPURLs(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seed := func(name, version, filename, source string, cached bool) { + t.Helper() + pkgPURL := "pkg:npm/" + name + versionPURL := pkgPURL + "@" + version + if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: "npm", Name: name}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + art := &Artifact{VersionPURL: versionPURL, Filename: filename, UpstreamURL: "https://example.com/" + filename, Source: source} + if cached { + art.StoragePath = sql.NullString{String: "npm/" + name + "/" + version + "/" + filename, Valid: true} + art.FetchedAt = sql.NullTime{Time: now, Valid: true} + } + if err := db.UpsertArtifact(art); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + } + seed("lib", "1.0.0", "lib-1.0.0.tgz", "", true) + seed("lib", "2.0.0", "lib-2.0.0.tgz", "", true) + seed("lib", "2.0.0", "lib-2.0.0-extra.tgz", "", true) + seed("lib", "3.0.0", "lib-3.0.0.tgz", "", false) + seed("lib", "4.0.0", "lib-4.0.0.tgz", "local", true) + seed("other", "9.0.0", "other-9.0.0.tgz", "", true) + + got, err := db.CachedVersionPURLs("pkg:npm/lib") + if err != nil { + t.Fatalf("CachedVersionPURLs: %v", err) + } + slices.Sort(got) + if want := []string{"pkg:npm/lib@1.0.0", "pkg:npm/lib@2.0.0"}; !slices.Equal(got, want) { + t.Errorf("CachedVersionPURLs() = %v, want %v", got, want) + } + }) +} diff --git a/internal/database/schema.go b/internal/database/schema.go index f0ec651..bca37bd 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -75,7 +75,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at DATETIME, updated_at DATETIME, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at DATETIME ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -128,6 +129,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( content_type TEXT, content_encoding TEXT, content_digest TEXT, + upstream_url TEXT, size INTEGER, last_modified DATETIME, fetched_at DATETIME, @@ -154,6 +156,40 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS metadata_snapshots ( + id INTEGER PRIMARY KEY, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + storage_path TEXT NOT NULL, + content_type TEXT, + content_encoding TEXT, + content_digest TEXT NOT NULL, + size INTEGER NOT NULL, + valid_from_ms INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path); + +CREATE TABLE IF NOT EXISTS projects ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at DATETIME NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name); +CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at); + +CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) +); +CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at DATETIME NOT NULL @@ -218,7 +254,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at TIMESTAMP, updated_at TIMESTAMP, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at TIMESTAMP ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -271,6 +308,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( content_type TEXT, content_encoding TEXT, content_digest TEXT, + upstream_url TEXT, size BIGINT, last_modified TIMESTAMP, fetched_at TIMESTAMP, @@ -297,6 +335,40 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms BIGINT NOT NULL ); +CREATE TABLE IF NOT EXISTS metadata_snapshots ( + id SERIAL PRIMARY KEY, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + storage_path TEXT NOT NULL, + content_type TEXT, + content_encoding TEXT, + content_digest TEXT NOT NULL, + size BIGINT NOT NULL, + valid_from_ms BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path); + +CREATE TABLE IF NOT EXISTS projects ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at TIMESTAMP NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name); +CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at); + +CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) +); +CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at TIMESTAMP NOT NULL @@ -320,7 +392,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at DATETIME, updated_at DATETIME, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at DATETIME ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -345,7 +418,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at TIMESTAMP, updated_at TIMESTAMP, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at TIMESTAMP ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -468,6 +542,10 @@ var migrations = []migration{ {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, + {"020_add_metadata_snapshots", migrateAddMetadataSnapshots}, + {"021_add_projects", migrateAddProjects}, + {"022_add_artifact_mirrored_at", migrateAddArtifactMirroredAt}, + {"023_add_metadata_upstream_url", migrateAddMetadataUpstreamURL}, } // isTableNotFound returns true if the error indicates a missing table. @@ -728,6 +806,22 @@ func migrateAddMetadataLink(db *DB) error { return nil } +// migrateAddMetadataUpstreamURL adds the URL a metadata document was +// fetched from, which the mirror refresh fetches it from again. +func migrateAddMetadataUpstreamURL(db *DB) error { + hasColumn, err := db.HasColumn("metadata_cache", "upstream_url") + if err != nil { + return fmt.Errorf("checking metadata_cache upstream_url column: %w", err) + } + if hasColumn { + return nil + } + if _, err := db.Exec("ALTER TABLE metadata_cache ADD COLUMN upstream_url TEXT"); err != nil { + return fmt.Errorf("adding metadata_cache upstream_url column: %w", err) + } + return nil +} + func migrateAddMetadataContentEncoding(db *DB) error { hasColumn, err := db.HasColumn("metadata_cache", "content_encoding") if err != nil { @@ -820,6 +914,41 @@ func migrateAddUpstreamHolds(db *DB) error { return nil } +// migrateAddProjects creates the tables of the project inventory: the +// projects that uploaded an SBOM and the package versions each one uses. +func migrateAddProjects(db *DB) error { + id, ts := "INTEGER PRIMARY KEY", sqliteDatetime + if db.dialect == DialectPostgres { + id, ts = "SERIAL PRIMARY KEY", postgresTimestamp + } + statements := []string{ + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS projects ( + id %s, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at %s NOT NULL + )`, id, ts), + `CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name)`, + `CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at)`, + `CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) + )`, + `CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version)`, + } + for _, stmt := range statements { + if _, err := db.Exec(stmt); err != nil { + return fmt.Errorf("creating project tables: %w", err) + } + } + return nil +} + // migrateAddPendingDeletesQueuedAtIndex indexes the column reclaim orders // the queue by, which grows large after a retention sweep. func migrateAddPendingDeletesQueuedAtIndex(db *DB) error { @@ -850,6 +979,36 @@ func migrateAddArtifactFilenameIndex(db *DB) error { return nil } +// migrateAddMetadataSnapshots creates the table of kept earlier versions of +// metadata documents, which dated snapshot requests are served from. +func migrateAddMetadataSnapshots(db *DB) error { + pk, integer := "id INTEGER PRIMARY KEY", "INTEGER" + if db.dialect == DialectPostgres { + pk, integer = "id SERIAL PRIMARY KEY", "BIGINT" + } + statements := []string{ + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS metadata_snapshots ( + %s, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + storage_path TEXT NOT NULL, + content_type TEXT, + content_encoding TEXT, + content_digest TEXT NOT NULL, + size %s NOT NULL, + valid_from_ms %s NOT NULL + )`, pk, integer, integer), + `CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms)`, + `CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path)`, + } + for _, query := range statements { + if _, err := db.Exec(query); err != nil { + return fmt.Errorf("creating metadata_snapshots: %w", err) + } + } + return nil +} + // migrateAddMetadataDates creates the table of Last-Modified dates for // rewritten metadata, which replicas sharing the database agree on. func migrateAddMetadataDates(db *DB) error { @@ -945,6 +1104,26 @@ func migrateAddArtifactSource(db *DB) error { return nil } +// migrateAddArtifactMirroredAt records when a mirror run last found an +// artifact cached, which retention counts as use and LRU eviction doesn't. +func migrateAddArtifactMirroredAt(db *DB) error { + hasCol, err := db.HasColumn("artifacts", "mirrored_at") + if err != nil { + return fmt.Errorf("checking column mirrored_at: %w", err) + } + if hasCol { + return nil + } + ts := sqliteDatetime + if db.dialect == DialectPostgres { + ts = postgresTimestamp + } + if _, err := db.Exec(fmt.Sprintf("ALTER TABLE artifacts ADD COLUMN mirrored_at %s", ts)); err != nil { + return fmt.Errorf("adding column mirrored_at to artifacts: %w", err) + } + return nil +} + // migrateAddArtifactManifest stores the manifest read from a pushed archive, // which the metadata of its ecosystem lists the version with. func migrateAddArtifactManifest(db *DB) error { diff --git a/internal/database/types.go b/internal/database/types.go index fb49a1c..4e31a68 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -151,6 +151,9 @@ type Artifact struct { UpdatedAt time.Time `db:"updated_at" json:"updated_at"` Source string `db:"source" json:"source"` Manifest sql.NullString `db:"manifest" json:"manifest,omitempty"` + // MirroredAt is when a mirror run last found the artifact cached. It + // keeps the artifact from retention but not from LRU eviction. + MirroredAt sql.NullTime `db:"mirrored_at" json:"-"` } func (a *Artifact) sourceOrUpstream() string { @@ -184,12 +187,15 @@ type MetadataCacheEntry struct { ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"` ContentEncoding sql.NullString `db:"content_encoding" json:"content_encoding,omitempty"` ContentDigest sql.NullString `db:"content_digest" json:"content_digest,omitempty"` - Size sql.NullInt64 `db:"size" json:"size,omitempty"` - LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` - FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` - LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + // UpstreamURL is where the document was fetched from. Rows written + // before it was recorded, and by writers that don't know it, have none. + UpstreamURL sql.NullString `db:"upstream_url" json:"upstream_url,omitempty"` + Size sql.NullInt64 `db:"size" json:"size,omitempty"` + LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` + FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` + LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } // Vulnerability represents a cached vulnerability record. diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go index 76f2de5..eea8ee3 100644 --- a/internal/denylist/denylist.go +++ b/internal/denylist/denylist.go @@ -40,6 +40,13 @@ func New(packages []string) (*Policy, error) { p.packages[key] = make(map[string]bool) } p.packages[key][id.version] = true + // Debian and RPM filenames leave the epoch out, so an entry with + // one also matches the version without it. + if pkg.Type == "deb" || pkg.Type == "rpm" { + if _, version, ok := strings.Cut(id.version, ":"); ok && version != "" { + p.packages[key][version] = true + } + } } return p, nil } @@ -128,6 +135,9 @@ func genericSwiftRegistry(pkg *purl.PURL) (scope, name string, ok bool) { // identify compares PURLs in purl's comparison form. The generic // swift-registry spelling is the proxy's own, so it is mapped here first. +// The namespace of a distro type names the distribution, which handlers +// include for some types and not others, so it is dropped as retention and +// the inventory do. func identify(pkg *purl.PURL) identity { if scope, name, ok := genericSwiftRegistry(pkg); ok { pkg = purl.New("swift", scope, name, pkg.Version, nil) @@ -138,6 +148,10 @@ func identify(pkg *purl.PURL) identity { // An entry without a channel matches every channel. channel = c.Qualifier("channel") } - key := purl.New(c.Type, c.Namespace, c.Name, "", nil).String() + namespace := c.Namespace + if packageurl.DistroType(c.Type) { + namespace = "" + } + key := purl.New(c.Type, namespace, c.Name, "", nil).String() return identity{key: packageurl.PolicyPackageKey(key), channel: channel, version: c.Version} } diff --git a/internal/denylist/denylist_test.go b/internal/denylist/denylist_test.go index a110608..03edc99 100644 --- a/internal/denylist/denylist_test.go +++ b/internal/denylist/denylist_test.go @@ -90,3 +90,38 @@ func TestEcosystemSpellings(t *testing.T) { t.Error("pypi versions not found under another spelling") } } + +// TestDistroNamespaces checks that the distribution namespace of a distro +// package is ignored on both sides, as handlers build some of these PURLs +// without one and some with one. +func TestDistroNamespaces(t *testing.T) { + p, err := New([]string{"pkg:deb/debian/curl@8.5.0-2", "pkg:rpm/fedora/curl@8.6.0-1.fc40", "pkg:apk/busybox@1.36.1-r15", "pkg:alpm/glibc@2.39-1"}) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"pkg:deb/curl@8.5.0-2", "pkg:rpm/curl@8.6.0-1.fc40", "pkg:apk/alpine/busybox@1.36.1-r15", "pkg:alpm/arch/glibc@2.39-1"} { + if !p.Denied(value) { + t.Errorf("not denied: %s", value) + } + } + if p.Denied("pkg:deb/curl@8.5.0-3") { + t.Error("other version denied") + } +} + +// TestDistroEpochs checks that a deb or rpm entry with an epoch matches the +// version as it appears in a filename, which leaves the epoch out. +func TestDistroEpochs(t *testing.T) { + p, err := New([]string{"pkg:deb/foo@1:2.3-1", "pkg:rpm/bar@2:1.0-1.fc40"}) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"pkg:deb/foo@2.3-1", "pkg:deb/foo@1:2.3-1", "pkg:rpm/bar@1.0-1.fc40"} { + if !p.Denied(value) { + t.Errorf("not denied: %s", value) + } + } + if p.Denied("pkg:deb/foo@2.3-2") { + t.Error("other version denied") + } +} diff --git a/internal/gitcache/budget.go b/internal/gitcache/budget.go index 9d4135e..2b0c228 100644 --- a/internal/gitcache/budget.go +++ b/internal/gitcache/budget.go @@ -2,6 +2,7 @@ package gitcache import ( "context" + "errors" "sync/atomic" "time" ) @@ -29,8 +30,15 @@ type reservation struct { // cut, guarded by c.mu, marks a fetch stopped because the cache could // not make room for it. cut bool + // ended, guarded by c.mu, marks a reservation given back. A read + // relayed after that, by a git helper the fetch left behind, reserves + // nothing. + ended bool } +// errReservationEnded is returned by cover once the fetch has ended. +var errReservationEnded = errors.New("fetch has ended") + func (c *Cache) newReservation() *reservation { return &reservation{c: c} } @@ -48,6 +56,10 @@ func (r *reservation) cover(ctx context.Context, relayed int64) error { var timeout <-chan time.Time for { c.mu.Lock() + if r.ended { + c.mu.Unlock() + return errReservationEnded + } if r.cut { c.mu.Unlock() return ErrCacheFull @@ -113,6 +125,7 @@ func (r *reservation) release() { func (r *reservation) releaseLocked() { c := r.c + r.ended = true n := r.reserved.Swap(0) if r.cut { r.cut = false diff --git a/internal/gitcache/budget_test.go b/internal/gitcache/budget_test.go index 506ef2f..8ef58fa 100644 --- a/internal/gitcache/budget_test.go +++ b/internal/gitcache/budget_test.go @@ -1,10 +1,15 @@ package gitcache import ( + "bytes" "context" "crypto/rand" "errors" + "io" "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" "os" "path/filepath" "sync" @@ -216,3 +221,91 @@ func TestConcurrentClonesStayWithinMaxSize(t *testing.T) { t.Errorf("left %d entries in the temporary directory", len(entries)) } } + +// TestCoverAfterReleaseReservesNothing has a relayed read arrive after the +// fetch gave back its reservation, as one from an orphaned git helper would. +func TestCoverAfterReleaseReservesNothing(t *testing.T) { + tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) + res := tc.newReservation() + if err := res.cover(t.Context(), 10); err != nil { + t.Fatal(err) + } + res.release() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _ = res.cover(ctx, 20) + if reserved, _ := tc.budget(); reserved != 0 { + t.Errorf("reserved %d after release", reserved) + } +} + +// TestCancelledFetchStopsRelaying cancels a fetch while the upstream is +// midway through sending the pack: the upstream connection is closed, and +// nothing stays reserved when the rest arrives. +func TestCancelledFetchStopsRelaying(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + upstreamURL, err := url.Parse(upstream.server.URL) + if err != nil { + t.Fatal(err) + } + arrived := make(chan struct{}) + release := make(chan struct{}) + result := make(chan string, 1) + forward := httputil.NewSingleHostReverseProxy(upstreamURL) + front := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + r.Body = io.NopCloser(bytes.NewReader(body)) + if r.Method != http.MethodPost || !bytes.Contains(body, []byte("command=fetch")) { + forward.ServeHTTP(w, r) + return + } + rec := httptest.NewRecorder() + forward.ServeHTTP(rec, r.Clone(context.Background())) + pack := rec.Body.Bytes() + w.Header().Set("Content-Type", rec.Header().Get("Content-Type")) + _, _ = w.Write(pack[:len(pack)/2]) + w.(http.Flusher).Flush() + close(arrived) + select { + case <-r.Context().Done(): + result <- "closed" + case <-release: + _, _ = w.Write(pack[len(pack)/2:]) + w.(http.Flusher).Flush() + result <- "sent" + } + })) + defer front.Close() + defer close(release) + tc := newTestCache(t, front.URL, func(o *Options) { o.MaxSize = 1 << 30 }) + + dir := t.TempDir() + if _, err := tc.run(t.Context(), dir, "init", "--quiet", "--bare"); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + res := tc.newReservation() + done := make(chan error, 1) + go func() { done <- tc.fetch(ctx, dir, front.URL+"/repo", 0, res, nil, branchesAndTags...) }() + <-arrived + waitFor(t, func() bool { reserved, _ := tc.budget(); return reserved > 0 }) + cancel() + if err := <-done; err == nil { + t.Error("cancelled fetch succeeded") + } + res.release() + + select { + case <-result: + case <-time.After(10 * time.Second): + t.Error("upstream connection still open after the fetch was cancelled") + release <- struct{}{} + <-result + } + // Give a relay still running the time to read what the upstream sent. + time.Sleep(200 * time.Millisecond) + if reserved, _ := tc.budget(); reserved != 0 { + t.Errorf("reserved %d after the fetch ended", reserved) + } +} diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index eb1578f..54c97d0 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -95,6 +95,9 @@ type Cache struct { refCheckTimeout time.Duration lowSpeedTime int maxRequestBody int64 + // foldCase is set where the cache directory ignores case, so keys that + // differ only in case, which name one directory, are one repository. + foldCase bool ctx context.Context cancel context.CancelFunc @@ -121,14 +124,17 @@ type Cache struct { failedWants map[string]time.Time } -// repo is a cached repository, or one being cloned. users, refreshing and -// the map entry itself are guarded by Cache.mu. +// repo is a cached repository, or one being cloned. users, refreshing, +// refused and the map entry itself are guarded by Cache.mu. type repo struct { key string dir string users int refreshing *refreshCall + // refused is the error of the last refresh when the upstream reported + // the repository missing or refused it, until a ref check succeeds. + refused error // writes serializes git fetches into the repository. writes chan struct{} @@ -214,6 +220,7 @@ func New(opts Options) (*Cache, error) { tooLarge: make(map[string]time.Time), failedWants: make(map[string]time.Time), freed: make(chan struct{}), + foldCase: caseInsensitive(dir), } if opts.refCheckTimeout > 0 { c.refCheckTimeout = opts.refCheckTimeout @@ -284,7 +291,7 @@ func (c *Cache) load() error { return err } rel, _ := filepath.Rel(c.dir, path) - rp := newRepo(filepath.ToSlash(rel), path) + rp := newRepo(c.repoKey(filepath.ToSlash(rel)), path) rp.size.Store(dirSize(path)) rp.lastChecked.Store(stampTime(path, checkedStamp)) rp.lastUsed.Store(stampTime(path, usedStamp)) @@ -352,15 +359,25 @@ func (c *Cache) removeRepointed() { func (c *Cache) acquire(key string) *repo { c.mu.Lock() defer c.mu.Unlock() - rp := c.repos[key] + rk := c.repoKey(key) + rp := c.repos[rk] if rp == nil { - rp = newRepo(key, filepath.Join(c.dir, filepath.FromSlash(key))) - c.repos[key] = rp + rp = newRepo(rk, filepath.Join(c.dir, filepath.FromSlash(key))) + c.repos[rk] = rp } rp.users++ return rp } +// repoKey is the key a repository is tracked under: key itself, or key +// lowercased where the cache directory ignores case. +func (c *Cache) repoKey(key string) string { + if c.foldCase { + return strings.ToLower(key) + } + return key +} + func (c *Cache) release(rp *repo) { now := time.Now() rp.lastUsed.Store(now.UnixNano()) @@ -511,3 +528,17 @@ func touch(dir, name string, t time.Time) { } } } + +// caseInsensitive reports whether the file system holding dir ignores case +// in names, as macOS and Windows do by default. +func caseInsensitive(dir string) bool { + f, err := os.CreateTemp(dir, ".gitcache-case-") + if err != nil { + return false + } + name := f.Name() + _ = f.Close() + defer func() { _ = os.Remove(name) }() + _, err = os.Stat(filepath.Join(dir, strings.ToUpper(filepath.Base(name)))) + return err == nil +} diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 4d43c5d..2b2e388 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "maps" + "math" "net/http" "net/http/httptest" "net/http/httputil" @@ -786,3 +787,248 @@ func TestCloseTwice(t *testing.T) { t.Errorf("closing twice: %v", err) } } + +// lsRefs sends a protocol v2 ls-refs request for "repo" straight to +// git-upload-pack, as a client may without asking for info/refs first, and +// returns the status. +func (tc *testCache) lsRefs(t *testing.T) int { + t.Helper() + body := pktLine("command=ls-refs\n") + pktDelim + pktLine("peel\n") + pktFlush + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, tc.url("repo")+"/git-upload-pack", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", requestType) + req.Header.Set("Git-Protocol", "version=2") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + return resp.StatusCode +} + +// TestUploadPackChecksCopyPastStaleMaxAge sends upload-pack requests without +// info/refs: a copy not checked within stale_max_age is checked first. +func TestUploadPackChecksCopyPastStaleMaxAge(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + if status := tc.lsRefs(t); status != http.StatusOK { + t.Fatalf("ls-refs on a fresh copy: status %d", status) + } + + waitIdle(t, tc.Cache) + rp := tc.acquire("up/repo.git") + rp.lastChecked.Store(time.Now().Add(-2 * time.Hour).UnixNano()) + tc.release(rp) + upstream.status.Store(http.StatusNotFound) + if status := tc.lsRefs(t); status == http.StatusOK { + t.Errorf("served a copy older than stale_max_age to ls-refs") + } +} + +// TestUploadPackRefusedAfterUpstreamRefusal has the upstream refuse a +// refresh: upload-pack requests are refused too until a check succeeds. +func TestUploadPackRefusedAfterUpstreamRefusal(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusForbidden) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("served the cached copy after 403:\n%s", out) + } + if status := tc.lsRefs(t); status == http.StatusOK { + t.Errorf("ls-refs served after the upstream refused the repository") + } + + upstream.status.Store(0) + tc.clone(t, "repo") + if status := tc.lsRefs(t); status != http.StatusOK { + t.Errorf("ls-refs after a successful check: status %d", status) + } +} + +// TestCloneChecksUpstreamOnce clones with a stale_max_age of 0: the +// upload-pack requests after info/refs do not check the upstream again. +func TestCloneChecksUpstreamOnce(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.StaleMaxAge = 0 }) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + checks := func() int32 { return upstream.requests.Load() - upstream.gitRequests.Load() } + before := checks() + resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + perCheck := checks() - before + + before = checks() + tc.clone(t, "repo") + if got := checks() - before; got != perCheck { + t.Errorf("clone made %d upstream check requests, want %d", got, perCheck) + } +} + +// TestInfoRefsRefusedWhileRefusalStands checks that once the upstream +// refused a repository, a later check that runs past RefreshWait, which +// would otherwise serve the cached copy, doesn't advertise its refs. +func TestInfoRefsRefusedWhileRefusalStands(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = 100 * time.Millisecond }) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusForbidden) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("served the cached copy after 403:\n%s", out) + } + upstream.status.Store(0) + release := upstream.holdRequests() + defer release() + // Protocol v0 advertises the refs in the info/refs response itself. + if out, err := tryGit(t, "", "-c", "protocol.version=0", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("advertised refs while the refusal stands:\n%s", out) + } +} + +// refuse has the upstream refuse "repo" with a 403 and checks that the +// cached copy is no longer served. +func (tc *testCache) refuse(t *testing.T, upstream *fakeUpstream) { + t.Helper() + upstream.status.Store(http.StatusForbidden) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("served the cached copy after 403:\n%s", out) + } + upstream.status.Store(0) +} + +// TestUploadPackClearsRefusal has a client that only sends upload-pack +// requests: once the upstream allows the repository again, it is served. +func TestUploadPackClearsRefusal(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + tc.refuse(t, upstream) + + if status := tc.lsRefs(t); status != http.StatusOK { + t.Errorf("ls-refs after the upstream allowed the repository again: status %d", status) + } +} + +// TestRefCheckClearsRefusal has the upstream list the refs again after a +// refusal but fail the fetch that follows: the check alone clears the +// refusal, and the cached copy is served. +func TestRefCheckClearsRefusal(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + tc.refuse(t, upstream) + + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + upstream.fetchStatus.Store(http.StatusInternalServerError) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err != nil { + t.Errorf("cached copy refused after the upstream listed its refs: %v\n%s", err, out) + } + if status := tc.lsRefs(t); status != http.StatusOK { + t.Errorf("ls-refs after the upstream listed its refs: status %d", status) + } +} + +// TestHugeStaleMaxAgeUploadPackChecksNothing sets stale_max_age near the +// largest duration: upload-pack requests on a fresh copy do not check the +// upstream. +func TestHugeStaleMaxAgeUploadPackChecksNothing(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.StaleMaxAge = math.MaxInt64 - time.Second }) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + before := upstream.requests.Load() + if status := tc.lsRefs(t); status != http.StatusOK { + t.Fatalf("ls-refs: status %d", status) + } + waitIdle(t, tc.Cache) + if got := upstream.requests.Load() - before; got != 0 { + t.Errorf("ls-refs made %d upstream requests, want 0", got) + } +} + +// TestFetchRedirectRefusesRepository has the upstream list the refs but +// redirect the fetch that follows: the repository is refused, as when the +// ref listing is redirected. +func TestFetchRedirectRefusesRepository(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + upstream.fetchStatus.Store(http.StatusFound) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("served the cached copy after the fetch was redirected:\n%s", out) + } + if status := tc.lsRefs(t); status == http.StatusOK { + t.Errorf("ls-refs served after the fetch was redirected") + } +} + +// TestCaseOnlyDifferentKeysShareOneRepository checks that, where the file +// system ignores case, keys that differ only in case are one repository, so +// a user of one spelling keeps the directory from being evicted under it. +func TestCaseOnlyDifferentKeysShareOneRepository(t *testing.T) { + if !caseInsensitive(t.TempDir()) { + t.Skip("file system is case-sensitive") + } + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + rp := tc.acquire("up/REPO.git") + defer tc.release(rp) + if other := tc.repo("up/repo.git"); other != rp { + t.Error("up/REPO.git and up/repo.git are separate repositories") + } + tc.mu.Lock() + tc.opts.MaxSize = 1 + tc.mu.Unlock() + tc.evict() + if !rp.exists() { + t.Error("repository in use as up/REPO.git was evicted") + } +} + +// TestFoldCaseKeysShareOneEntry checks key folding on any file system by +// setting foldCase directly. +func TestFoldCaseKeysShareOneEntry(t *testing.T) { + tc := newTestCache(t, newFakeUpstream(t).server.URL, nil) + tc.foldCase = true + + upper := tc.acquire("up/REPO.git") + defer tc.release(upper) + lower := tc.acquire("up/repo.git") + defer tc.release(lower) + if upper != lower { + t.Error("up/REPO.git and up/repo.git are separate repositories") + } + tc.mu.Lock() + n := len(tc.repos) + tc.mu.Unlock() + if n != 1 { + t.Errorf("%d repository entries, want 1", n) + } +} diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go index 55b7333..8e6c2e5 100644 --- a/internal/gitcache/helpers_test.go +++ b/internal/gitcache/helpers_test.go @@ -75,8 +75,10 @@ type fakeUpstream struct { root string server *httptest.Server - // status, when set, is returned instead of serving the request. - status atomic.Int32 + // status, when set, is returned instead of serving the request, and + // fetchStatus instead of serving git's own upload-pack requests. + status atomic.Int32 + fetchStatus atomic.Int32 // requireAuth, when set, is the Authorization every request must carry. requireAuth string // v0Only drops Git-Protocol, so only protocol v0 is spoken. @@ -178,22 +180,7 @@ func newFakeUpstream(t *testing.T) *fakeUpstream { if u.v0Only { r.Header.Del("Git-Protocol") } - if r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { - u.holdMu.Lock() - fetchHold, fetchArrived, fetchOnce := u.fetchHold, u.fetchArrived, u.fetchOnce - u.holdMu.Unlock() - if fetchHold != nil { - fetchOnce.Do(func() { close(fetchArrived) }) - <-fetchHold - } - } - if delay := u.delayFetches.Load(); delay > 0 && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { - time.Sleep(time.Duration(delay)) - } - if u.stallFetches.Load() && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { - // The server notices the client leaving only once the body is read. - _, _ = io.Copy(io.Discard, r.Body) - <-r.Context().Done() + if r.UserAgent() != gitUserAgent && r.Method == http.MethodPost && u.interceptFetch(w, r) { return } if r.Method == http.MethodPost { @@ -205,6 +192,33 @@ func newFakeUpstream(t *testing.T) *fakeUpstream { return u } +// interceptFetch applies the fetch knobs to one of git's own upload-pack +// requests, and reports whether it answered the request. +func (u *fakeUpstream) interceptFetch(w http.ResponseWriter, r *http.Request) bool { + u.holdMu.Lock() + fetchHold, fetchArrived, fetchOnce := u.fetchHold, u.fetchArrived, u.fetchOnce + u.holdMu.Unlock() + if fetchHold != nil { + fetchOnce.Do(func() { close(fetchArrived) }) + <-fetchHold + } + if delay := u.delayFetches.Load(); delay > 0 { + time.Sleep(time.Duration(delay)) + } + if u.stallFetches.Load() { + // The server notices the client leaving only once the body is read. + _, _ = io.Copy(io.Discard, r.Body) + <-r.Context().Done() + return true + } + if status := u.fetchStatus.Load(); status != 0 { + w.Header().Set("Location", u.server.URL+"/elsewhere") + http.Error(w, "unavailable", int(status)) + return true + } + return false +} + // holdFetches holds git's own upload-pack requests until release is called. // arrived is closed when the first one reaches the upstream. func (u *fakeUpstream) holdFetches() (arrived <-chan struct{}, release func()) { @@ -302,6 +316,7 @@ func (u *fakeUpstream) bare() string { type testCache struct { *Cache server *httptest.Server + logs *lockedBuffer } func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *testCache { @@ -333,7 +348,7 @@ func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *t server.Close() _ = c.Close() }) - return &testCache{Cache: c, server: server} + return &testCache{Cache: c, server: server, logs: logs} } // testFront serves /git/up/{repo}/info/refs and git-upload-pack from c, as diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go new file mode 100644 index 0000000..6e45d43 --- /dev/null +++ b/internal/gitcache/mirror.go @@ -0,0 +1,253 @@ +package gitcache + +import ( + "context" + "errors" + "fmt" + "os/exec" + "slices" + "strings" + "time" +) + +// MirrorResult is what Mirror did. +type MirrorResult struct { + // Cached reports that the repository, and the commit if a version was + // given, were cached already. + Cached bool + // Bytes is how much the cached repository grew. + Bytes int64 + // Commit is the commit the version names, empty without a version. + Commit string + // Submodules are the submodules Commit records, sorted by path. + Submodules []Submodule +} + +// Submodule is a submodule a commit records: its path, the URL .gitmodules +// gives for it, which may be relative to the repository's, and the commit +// the gitlink pins. +type Submodule struct { + Path, URL, Commit string +} + +// Mirror brings the repository cached under key up to date with repoURL, +// cloning it if needed, and waits for that to finish. version, if given, is +// a full commit ID or a tag. A commit is made sure to be cached: one on no +// branch or tag is fetched by ID when fetchWants allows it (see +// OpenOptions.FetchWants), and is kept as long as a commit a client fetched +// by ID. A tag is cached with the repository's tags. +func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) { + if lower := strings.ToLower(version); isObjectID(lower) { + version = lower + } + tag := version != "" && !isObjectID(version) + if tag { + if err := c.checkTagName(ctx, version); err != nil { + return MirrorResult{}, err + } + } + lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants, NoCheck: true}) + if err != nil { + return MirrorResult{}, err + } + defer lease.Release() + rp := lease.repo + cached := lease.hit + before := int64(0) + var refreshErr error + // Open cloned a repository that wasn't cached; one that was is + // refreshed here, waiting for the fetch a client would not wait for. + if cached { + before = rp.size.Load() + refresh := c.startRefresh(rp, repoURL) + select { + case <-refresh.done: + refreshErr = refresh.err + case <-ctx.Done(): + return MirrorResult{}, ctx.Err() + } + } + + // A refresh past max_repo_size gets the repository dropped once it is + // released, commit and all. + if errors.Is(refreshErr, ErrTooLarge) { + return MirrorResult{}, refreshErr + } + commit := version + switch { + case tag: + if commit, err = c.resolveTag(ctx, rp.dir, version); err != nil { + if refreshErr != nil { + // The tag may be one the failed refresh would have brought. + return MirrorResult{}, refreshErr + } + if errors.Is(err, ErrNotFound) && strings.Trim(strings.ToLower(version), "0123456789abcdef") == "" { + // Likely an abbreviated commit ID. + return MirrorResult{}, fmt.Errorf("%s is not a full commit ID or tag: %w", version, ErrNotFound) + } + return MirrorResult{}, err + } + if refreshErr != nil { + // Tags can move; this is where it pointed when last fetched. + c.logger.Warn("git cache: refresh failed, mirroring the cached tag", "repository", key, "tag", version, "commit", commit, "error", refreshErr) + } + case version != "": + if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { + return MirrorResult{}, err + } + } + // A refresh that failed leaves the commit cached all the same. + if refreshErr != nil && version == "" { + return MirrorResult{}, refreshErr + } + var submodules []Submodule + if commit != "" { + if submodules, err = c.submodules(ctx, key, rp.dir, commit); err != nil { + return MirrorResult{}, err + } + } + grown := max(rp.size.Load()-before, 0) + return MirrorResult{Cached: cached && grown == 0, Bytes: grown, Commit: commit, Submodules: submodules}, nil +} + +// checkTagName refuses a version that is no valid tag name, such as one +// with .. or a revision suffix like ^{tree}. +func (c *Cache) checkTagName(ctx context.Context, name string) error { + if err := c.command(ctx, nil, "check-ref-format", "refs/tags/"+name).Run(); err != nil { + return fmt.Errorf("%q is neither a full commit ID nor a tag name", name) + } + return nil +} + +// resolveTag returns the commit tag points at. +func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) { + out, err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", "refs/tags/"+tag+"^{commit}").Output() + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 1 { + return "", fmt.Errorf("tag %s: %w", tag, ErrNotFound) + } + if err != nil { + return "", fmt.Errorf("resolving tag %s: %w", tag, err) + } + return strings.TrimSpace(string(out)), nil +} + +// submodules lists the submodules commit records: its gitlinks that +// .gitmodules names a URL for. +func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Submodule, error) { + err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", commit+":.gitmodules").Run() + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 1 && ctx.Err() == nil { + return nil, nil + } + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("looking for .gitmodules: %w", err) + } + // --list rather than --get-regexp, which fails when nothing matches. + out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list") + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + // The commit is cached; a .gitmodules git can't read only loses + // its submodules. + c.logger.Warn("git cache: cannot read .gitmodules, skipping submodules", "repository", key, "commit", commit, "error", err) + return nil, nil + } + paths, urls := map[string]string{}, map[string]string{} + for entry := range strings.SplitSeq(out, "\x00") { + key, value, ok := strings.Cut(entry, "\n") + section, ok2 := strings.CutPrefix(key, "submodule.") + name, field, dotted := strings.CutLast(section, ".") + if !ok || !ok2 || !dotted { + continue + } + switch field { + case "path": + paths[value] = name + case "url": + urls[name] = value + } + } + var pathspecs []string + for path, name := range paths { + if path != "" && urls[name] != "" { + pathspecs = append(pathspecs, path) + } + } + if len(pathspecs) == 0 { + return nil, nil + } + args := append([]string{"--literal-pathspecs", "ls-tree", "-z", "--full-tree", commit, "--"}, pathspecs...) + tree, err := c.run(ctx, dir, args...) + if err != nil { + return nil, err + } + var submodules []Submodule + for entry := range strings.SplitSeq(tree, "\x00") { + meta, path, ok := strings.Cut(entry, "\t") + fields := strings.Fields(meta) + if !ok || len(fields) != 3 || fields[0] != "160000" { + continue + } + if url := urls[paths[path]]; url != "" && isObjectID(fields[2]) { + submodules = append(submodules, Submodule{Path: path, URL: url, Commit: fields[2]}) + } + } + slices.SortFunc(submodules, func(a, b Submodule) int { return strings.Compare(a.Path, b.Path) }) + return submodules, nil +} + +// ensureCommit makes sure the repository has commit, fetching it by ID when +// it doesn't and fetchWants allows it. A commit on no branch or tag that is +// cached already gets a new wanted ref, so it is kept for wantedMaxAge from +// this run on. +func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) error { + missing, err := c.missingObjects(ctx, rp.dir, []string{commit}) + if err != nil { + return err + } + if len(missing) == 0 { + return c.renewWanted(ctx, rp, commit) + } + if !fetchWants { + return fmt.Errorf("commit %s: %w", commit, ErrNotFound) + } + if err := c.fetchMissingWants(ctx, rp, repoURL, missing); err != nil { + if ctx.Err() != nil || errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) { + return err + } + // The upstream refuses a commit it doesn't have. + return fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err) + } + // fetchMissingWants skips a commit that failed moments ago. + if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil { + return err + } + if len(missing) > 0 { + return fmt.Errorf("commit %s: %w", commit, ErrNotFound) + } + return nil +} + +// renewWanted writes a new wanted ref for commit when no branch or tag +// includes it. +func (c *Cache) renewWanted(ctx context.Context, rp *repo, commit string) error { + unlock, err := rp.lockWrites(ctx) + if err != nil { + return err + } + defer unlock() + unmerged, err := c.unmergedCommits(ctx, rp.dir, []string{commit}) + if err != nil || !unmerged[commit] { + return err + } + cmd := c.command(ctx, nil, "-C", rp.dir, "update-ref", wantedRef(commit, time.Now()), commit) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("renewing wanted ref: %w: %s", err, strings.TrimSpace(string(out))) + } + return nil +} diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go new file mode 100644 index 0000000..d016f2a --- /dev/null +++ b/internal/gitcache/mirror_test.go @@ -0,0 +1,415 @@ +package gitcache + +import ( + "context" + "errors" + "net/http" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +func TestMirrorClonesRepository(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached || got.Bytes <= 0 { + t.Errorf("first mirror = %+v, want fetched bytes", got) + } + if !strings.Contains(tc.cachedRefs(t), "refs/tags/v1") { + t.Errorf("cached refs %q lack the tag", tc.cachedRefs(t)) + } + + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } +} + +func TestMirrorFetchesCommitOnNoBranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("mirror = %+v, want fetched", got) + } + if refs := tc.wantedRefs(t); len(refs) != 1 { + t.Errorf("wanted refs %v, want one", refs) + } + + sessions := upstream.gitSessions.Load() + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + if n := upstream.gitSessions.Load() - sessions; n != 0 { + t.Errorf("second mirror fetched %d times, want none", n) + } +} + +func TestMirrorFindsCommitOnABranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + sha := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, false) + if err != nil || got.Cached { + t.Errorf("mirror = %+v, %v, want fetched", got, err) + } +} + +func TestMirrorDoesNotFetchByIDWhenTurnedOff(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, false) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror of a commit on no branch with fetching by ID off = %v, want ErrNotFound", err) + } + if refs := tc.wantedRefs(t); len(refs) != 0 { + t.Errorf("wanted refs %v, want none", refs) + } +} + +func TestMirrorReportsMissingCommit(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", strings.Repeat("ab", 20), true) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror of a commit the upstream lacks = %v, want ErrNotFound", err) + } +} + +func TestMirrorRefusesInvalidCommit(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "main", true); err == nil { + t.Error("mirrored a branch name as a commit") + } +} + +func TestMirrorCountsRefreshThatFetched(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true); err != nil { + t.Fatalf("Mirror: %v", err) + } + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached || got.Bytes <= 0 { + t.Errorf("mirror after a push = %+v, want fetched bytes", got) + } +} + +// TestMirrorRenewsWantedCommit checks mirroring a commit on no branch again +// makes its 7 days count from that run, not from the first fetch. +func TestMirrorRenewsWantedCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + cached := filepath.Join(tc.dir, "up", "repo.git") + for _, ref := range tc.wantedRefs(t) { + runGit(t, cached, "update-ref", "-d", ref) + } + old := wantedRef(sha, time.Now().Add(-wantedMaxAge+time.Hour)) + runGit(t, cached, "update-ref", old, sha) + + start := time.Now().Add(-time.Second) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil || !got.Cached { + t.Fatalf("second mirror = %+v, %v, want cached", got, err) + } + if !slices.ContainsFunc(tc.wantedRefs(t), func(ref string) bool { + return ref != old && wantedRefTime(t, ref, sha).After(start) + }) { + t.Errorf("wanted refs %v, want one written by the second mirror", tc.wantedRefs(t)) + } +} + +func TestMirrorDoesNotPinCommitOnABranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + sha := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + for range 2 { + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + } + if refs := tc.wantedRefs(t); len(refs) != 0 { + t.Errorf("wanted refs %v, want none for a commit on a branch", refs) + } +} + +func wantedRefTime(t *testing.T, ref, oid string) time.Time { + t.Helper() + created, got, _ := strings.Cut(strings.TrimPrefix(ref, wantedPrefix), "/") + unix, err := strconv.ParseInt(created, 10, 64) + if err != nil || got != oid { + return time.Time{} + } + return time.Unix(unix, 0) +} + +// TestMirrorReportsRefreshTooLarge checks a commit that is cached still +// fails to mirror when the refresh doesn't fit in max_repo_size, since the +// repository is dropped once released. +func TestMirrorReportsRefreshTooLarge(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + pinned := upstream.pushPullRequest(work, 1, randomBytes(100<<10)) + base := repoSize(t, upstream, "repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 150<<10 }) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", pinned, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + if err := os.WriteFile(filepath.Join(work, "grown.bin"), randomBytes(80<<10), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "grown.bin") + upstream.commit(work, "grown") + runGit(t, work, "push", "--quiet", "origin", "main") + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", pinned, true) + if !errors.Is(err, ErrTooLarge) { + t.Errorf("mirror with a refresh past max_repo_size = %v, want ErrTooLarge", err) + } +} + +func TestMirrorResolvesTag(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tagged := runGit(t, work, "rev-parse", "v1^{commit}") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Commit != tagged { + t.Errorf("commit %q, want %s the tag points at", got.Commit, tagged) + } +} + +func TestMirrorReportsMissingTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, version := range []string{"v9", "main"} { + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror at %q = %v, want ErrNotFound", version, err) + } + } +} + +func TestMirrorRefusesInvalidTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, version := range []string{"a..b", "v1^{tree}", "v1~1", "v 1", "v1:x"} { + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) + if err == nil || errors.Is(err, ErrNotFound) { + t.Errorf("mirror at %q = %v, want an invalid version error", version, err) + } + } +} + +func TestMirrorListsSubmodules(t *testing.T) { + upstream := newFakeUpstream(t) + libWork := upstream.newRepo("lib") + libCommit := runGit(t, libWork, "rev-parse", "HEAD") + work := upstream.newRepo("repo") + gitmodules := "[submodule \"vendor/lib\"]\n\tpath = vendor/lib\n\turl = ../lib.git\n" + + "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + + "[submodule \"colon\"]\n\tpath = :colon\n\turl = ../lib.git\n" + if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { + t.Fatal(err) + } + otherCommit := strings.Repeat("ab", 20) + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") + // Git for Windows refuses ':' in paths unless NTFS protection is off. + runGit(t, work, "-c", "core.protectNTFS=false", "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",:colon") + runGit(t, work, "commit", "--quiet", "-m", "submodules") + runGit(t, work, "push", "--quiet", "origin", "main") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + want := []Submodule{ + {Path: ":colon", URL: "../lib.git", Commit: libCommit}, + {Path: "third_party/other", URL: "https://example.com/other.git", Commit: otherCommit}, + {Path: "vendor/lib", URL: "../lib.git", Commit: libCommit}, + } + if !slices.Equal(got.Submodules, want) { + t.Errorf("submodules %+v, want %+v", got.Submodules, want) + } + + // Without a version, nothing names a commit to read submodules from. + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("unversioned mirror = %+v, %v, want no submodules", got, err) + } +} + +// TestMirrorWithEmptyGitmodules checks a .gitmodules with no path or url, +// for which git config --get-regexp finds nothing, lists no submodules. +func TestMirrorWithEmptyGitmodules(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + for i, content := range []string{"", "[submodule \"x\"]\n\tbranch = main\n"} { + if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "--allow-empty", "-m", "gitmodules "+strconv.Itoa(i)) + } + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, rev := range []string{"HEAD~1", "HEAD"} { + commit := runGit(t, work, "rev-parse", rev) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err) + } + } +} + +// TestSubmodulesReportsCancel checks a cancelled context is reported, not +// read as a commit without .gitmodules. +func TestSubmodulesReportsCancel(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + tc := newTestCache(t, upstream.server.URL, nil) + + if _, err := tc.submodules(ctx, "up/repo.git", work, commit); !errors.Is(err, context.Canceled) { + t.Errorf("submodules with a cancelled context = %v, want context.Canceled", err) + } +} + +// TestMirrorSkipsBrokenGitmodules checks a .gitmodules git can't read, one +// with a syntax error or committed as a symlink, lists no submodules and +// logs a warning instead of failing the mirror. +func TestMirrorSkipsBrokenGitmodules(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + gitmodules := filepath.Join(work, ".gitmodules") + if err := os.WriteFile(gitmodules, []byte("[submodule \"x\"\n\tpath = x\n"), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "-m", "broken gitmodules") + if err := os.Remove(gitmodules); err != nil { + t.Fatal(err) + } + if err := os.Symlink("../outside/.gitmodules", gitmodules); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "-m", "symlinked gitmodules") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, rev := range []string{"HEAD~1", "HEAD"} { + commit := runGit(t, work, "rev-parse", rev) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err) + } + if !strings.Contains(tc.logs.String(), "commit="+commit) { + t.Errorf("no warning naming %s in the log", commit) + } + } +} + +func TestMirrorAcceptsUppercaseCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", strings.ToUpper(commit), true) + if err != nil || got.Commit != commit { + t.Errorf("mirror of the uppercase commit ID = %+v, %v, want commit %s", got, err, commit) + } +} + +func TestMirrorExplainsShortCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit[:12], true) + if !errors.Is(err, ErrNotFound) || !strings.Contains(err.Error(), "full commit ID") { + t.Errorf("mirror of a short commit ID = %v, want ErrNotFound saying it is no full commit ID", err) + } +} + +// TestMirrorWarnsOfStaleTag checks a tag served from the cache when the +// refresh fails is mirrored with a warning, since the tag may have moved. +func TestMirrorWarnsOfStaleTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true); err != nil { + t.Fatalf("Mirror: %v", err) + } + upstream.status.Store(http.StatusServiceUnavailable) + + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true); err != nil { + t.Fatalf("mirror of a cached tag with the upstream down: %v", err) + } + if !strings.Contains(tc.logs.String(), "tag=v1") { + t.Errorf("no warning naming the tag in the log:\n%s", tc.logs.String()) + } +} diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index 448b3d9..f467a3d 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -157,6 +157,9 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { } c.mu.Lock() rp.refreshing = nil + if call.err != nil && !servesStale(call.err) { + rp.refused = call.err + } c.mu.Unlock() c.release(rp) close(call.done) @@ -180,6 +183,11 @@ func (c *Cache) refresh(ctx context.Context, rp *repo, repoURL string, checked f if err != nil { return err } + // The upstream lets us read the repository again, even if the fetch + // below fails. + c.mu.Lock() + rp.refused = nil + c.mu.Unlock() if rp.exists() { err = c.update(ctx, rp, repoURL, remote, checked) } else { @@ -299,7 +307,7 @@ func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, res return ctx.Err() } - relayURL, route, unregister := c.relay.register(repoURL, limit, res) + relayURL, route, unregister := c.relay.register(ctx, repoURL, limit, res) defer unregister() args := append([]string{"fetch", "--quiet", "--no-write-fetch-head"}, flags...) _, err := c.run(ctx, dir, append(append(args, relayURL), refspecs...)...) @@ -318,6 +326,9 @@ func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, res if route.full.Load() { return ErrCacheFull } + if status := route.redirected.Load(); status != 0 { + return upstreamStatusError(int(status)) + } if err != nil && !route.answered.Load() { return fmt.Errorf("%w: %w", errUpstreamUnanswered, err) } diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go index 28f87f9..468879c 100644 --- a/internal/gitcache/relay.go +++ b/internal/gitcache/relay.go @@ -39,6 +39,8 @@ type relay struct { type relayRoute struct { repoURL string + // ctx ends when the fetch does. + ctx context.Context // limit caps the response bytes the route relays, 0 for no cap, and // budget, when set, holds space within MaxSize for them. exceeded and // full record which of them stopped the route. @@ -50,6 +52,9 @@ type relayRoute struct { // answered records that the upstream answered a git-upload-pack // request, so a failed fetch was not an outage. answered atomic.Bool + // redirected records the status of a redirect the upstream answered + // with, 0 for none. + redirected atomic.Int32 } func newRelay(client *http.Client, logger *slog.Logger) (*relay, error) { @@ -79,13 +84,15 @@ func (rl *relay) close() error { return rl.server.Close() } -// register adds a route to repoURL and returns the URL git fetches from and -// a function that removes the route. -func (rl *relay) register(repoURL string, limit int64, budget *reservation) (string, *relayRoute, func()) { +// register adds a route to repoURL for a fetch running with ctx and returns +// the URL git fetches from and a function that removes the route and stops +// its requests. +func (rl *relay) register(ctx context.Context, repoURL string, limit int64, budget *reservation) (string, *relayRoute, func()) { var b [relayIDBytes]byte _, _ = rand.Read(b[:]) id := hex.EncodeToString(b[:]) - route := &relayRoute{repoURL: repoURL, limit: limit, budget: budget} + ctx, cancel := context.WithCancel(ctx) + route := &relayRoute{repoURL: repoURL, ctx: ctx, limit: limit, budget: budget} rl.mu.Lock() rl.routes[id] = route @@ -95,6 +102,7 @@ func (rl *relay) register(repoURL string, limit int64, budget *reservation) (str rl.mu.Lock() delete(rl.routes, id) rl.mu.Unlock() + cancel() } } @@ -119,7 +127,10 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - out, err := http.NewRequestWithContext(r.Context(), r.Method, target, r.Body) + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + defer context.AfterFunc(route.ctx, cancel)() + out, err := http.NewRequestWithContext(ctx, r.Method, target, r.Body) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -133,13 +144,16 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { resp, err := rl.client.Do(out) if err != nil { - rl.logger.Warn("git relay: upstream request failed", "upstream", route.repoURL, "error", err) + if ctx.Err() == nil { + rl.logger.Warn("git relay: upstream request failed", "upstream", route.repoURL, "error", err) + } http.Error(w, "upstream request failed", http.StatusBadGateway) return } defer func() { _ = resp.Body.Close() }() // A redirect passed on would send git to the Location itself. if resp.StatusCode >= http.StatusMultipleChoices && resp.StatusCode < http.StatusBadRequest { + route.redirected.Store(int32(resp.StatusCode)) http.Error(w, "upstream redirect not followed", http.StatusBadGateway) return } @@ -151,12 +165,12 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", ct) } w.WriteHeader(resp.StatusCode) - if _, err := io.Copy(w, &routeReader{ctx: r.Context(), r: resp.Body, route: route}); err != nil { + if _, err := io.Copy(w, &routeReader{ctx: ctx, r: resp.Body, route: route}); err != nil { if r.Context().Err() != nil { // git went away; there is no one to tell. return } - if !route.exceeded.Load() && !route.full.Load() { + if ctx.Err() == nil && !route.exceeded.Load() && !route.full.Load() { rl.logger.Warn("git relay: copying upstream response", "upstream", route.repoURL, "error", err) } // Cut the connection so git fails rather than taking a response diff --git a/internal/gitcache/relay_test.go b/internal/gitcache/relay_test.go index ea1d063..309e727 100644 --- a/internal/gitcache/relay_test.go +++ b/internal/gitcache/relay_test.go @@ -28,7 +28,7 @@ func TestRelayOnlyForwardsFetchRequests(t *testing.T) { })) defer upstream.Close() rl := newTestRelay(t) - base, _, unregister := rl.register(upstream.URL+"/repo", 0, nil) + base, _, unregister := rl.register(t.Context(), upstream.URL+"/repo", 0, nil) refused := []struct{ method, url string }{ {http.MethodGet, rl.base + "/unknown/info/refs?service=git-upload-pack"}, @@ -79,7 +79,7 @@ func TestRelayStopsAtLimit(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = rl.close() }) - base, route, unregister := rl.register(upstream.URL+"/repo", limit, nil) + base, route, unregister := rl.register(t.Context(), upstream.URL+"/repo", limit, nil) defer unregister() resp, err := http.Post(base+"/git-upload-pack", requestType, strings.NewReader(pktFlush)) diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index bb5cfb0..5d80938 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -6,6 +6,7 @@ import ( "context" "errors" "io" + "math" "net/http" "os" "path/filepath" @@ -13,6 +14,7 @@ import ( "slices" "strings" "sync/atomic" + "time" ) // gitProtocolPattern is what a Git-Protocol header may hold, such as @@ -42,6 +44,11 @@ type OpenOptions struct { // commits the client may not read through this repository, such as // GitHub forks with credentials configured. FetchWants bool + // NoCheck leaves a cached repository as it is, for a caller that + // refreshes it itself. Otherwise a cached repository is refused when the + // last refresh found it gone or refused upstream, and refreshed first + // when it was not checked within StaleMaxAge. + NoCheck bool } // Open returns a lease on the repository cached under key, a relative path @@ -53,15 +60,57 @@ func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) } rp := c.acquire(key) hit := rp.exists() - if opts.Refresh || !hit { - if err := c.ensureFresh(ctx, rp, repoURL); err != nil { - c.release(rp) - return nil, err + var err error + switch { + case opts.Refresh || !hit: + err = c.ensureFresh(ctx, rp, repoURL) + if err == nil && hit && !opts.NoCheck { + // A refresh still running or failed serves the cached copy, + // but not one the upstream refused since and no check has + // cleared. + err = c.refusal(rp) } + case !opts.NoCheck: + err = c.checkServable(ctx, rp, repoURL) + } + if err != nil { + c.release(rp) + return nil, err } return &Lease{cache: c, repo: rp, repoURL: repoURL, hit: hit, fetchWants: opts.FetchWants}, nil } +// uploadPackGrace is how long past StaleMaxAge a copy is still served +// without a check to the upload-pack requests that follow a client's +// info/refs, so a clone checks the upstream once. +const uploadPackGrace = time.Minute + +// checkServable refreshes a cached repository first when the last refresh +// found it gone or refused upstream, or it was not checked recently, and +// refuses it while that refusal stands. +func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) error { + // The sum saturates rather than overflow for a huge StaleMaxAge. + maxAge := c.opts.StaleMaxAge + uploadPackGrace + if maxAge < c.opts.StaleMaxAge { + maxAge = math.MaxInt64 + } + if c.refusal(rp) == nil && time.Since(time.Unix(0, rp.lastChecked.Load())) <= maxAge { + return nil + } + if err := c.ensureFresh(ctx, rp, repoURL); err != nil { + return err + } + return c.refusal(rp) +} + +// refusal returns the error of the last refresh that found the repository +// gone or refused upstream, until a ref check succeeds. +func (c *Cache) refusal(rp *repo) error { + c.mu.Lock() + defer c.mu.Unlock() + return rp.refused +} + // Hit reports whether the repository was already cached when it was opened. func (l *Lease) Hit() bool { return l.hit diff --git a/internal/gitcache/wants.go b/internal/gitcache/wants.go index c715f2f..8755ba1 100644 --- a/internal/gitcache/wants.go +++ b/internal/gitcache/wants.go @@ -161,19 +161,10 @@ func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fe } var mergedErr error if fetched && len(keep) > 0 { - // rev-list prints what the wanted commits, read from stdin, reach - // and no branch or tag does, so a wanted commit it leaves out is - // merged. - cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") - cmd.Stdin = strings.NewReader(strings.Join(slices.Sorted(maps.Keys(keep)), "\n") + "\n") - out, err := cmd.Output() + unmerged, err := c.unmergedCommits(ctx, dir, slices.Sorted(maps.Keys(keep))) if err != nil { mergedErr = fmt.Errorf("listing unmerged wanted commits: %w", err) } else { - unmerged := make(map[string]bool) - for line := range strings.Lines(string(out)) { - unmerged[strings.TrimSpace(line)] = true - } for oid, ref := range keep { if !unmerged[oid] { drop = append(drop, ref) @@ -196,6 +187,24 @@ func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fe return mergedErr } +// unmergedCommits returns which of commits no cached branch or tag +// includes. +func (c *Cache) unmergedCommits(ctx context.Context, dir string, commits []string) (map[string]bool, error) { + // rev-list prints what the commits, read from stdin, reach and no + // branch or tag does, so a commit it leaves out is merged. + cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") + cmd.Stdin = strings.NewReader(strings.Join(commits, "\n") + "\n") + out, err := cmd.Output() + if err != nil { + return nil, err + } + unmerged := make(map[string]bool) + for line := range strings.Lines(string(out)) { + unmerged[strings.TrimSpace(line)] = true + } + return unmerged, nil +} + // withoutFailedWants drops the object IDs a recent fetch for the // repository could not get. func (c *Cache) withoutFailedWants(key string, oids []string) []string { diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 9024fe0..045dbac 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -6,6 +6,7 @@ import ( "net/http" "regexp" "strings" + "time" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" @@ -51,24 +52,39 @@ type APKHandler struct { proxy *Proxy proxyURL string repositories map[string]string + // snapshots: see WithSnapshots. + snapshots time.Duration } // NewAPKHandler creates an Alpine APK repository protocol handler. // When repositories is empty, a single repository named "alpine" pointing at // the official Alpine mirror is used. func NewAPKHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *APKHandler { - h := &APKHandler{ + return &APKHandler{ proxy: proxy, proxyURL: strings.TrimSuffix(proxyURL, "/"), - repositories: make(map[string]string, len(repositories)), + repositories: APKRepositories(repositories), } - for name, repositoryURL := range repositories { - h.repositories[name] = strings.TrimSuffix(repositoryURL, "/") +} + +// APKRepositories returns the upstream.apk repositories by name without +// trailing slashes, or the official Alpine mirror as "alpine" when none are +// configured. +func APKRepositories(configured map[string]string) map[string]string { + repositories := make(map[string]string, len(configured)) + for name, repositoryURL := range configured { + repositories[name] = strings.TrimSuffix(repositoryURL, "/") } - if len(h.repositories) == 0 { - h.repositories[defaultAPKRepositoryName] = defaultAPKUpstream + if len(repositories) == 0 { + repositories[defaultAPKRepositoryName] = defaultAPKUpstream } - return h + return repositories +} + +// APKCacheFilename returns the filename a package at path +// ({release}/{repo}/{arch}/{file}) of repository is cached under. +func APKCacheFilename(repository, path string) string { + return repository + "/" + path } // Routes returns the HTTP handler for APK requests. @@ -87,14 +103,18 @@ func (h *APKHandler) Routes() http.Handler { return } - repository, rest, ok := strings.Cut(path, "/") - upstreamURL, found := h.repositories[repository] + segment, rest, ok := strings.Cut(path, "/") + repository, upstreamURL, day, snapshot, found := h.resolveRepository(segment) if !ok || rest == "" || !found { http.NotFound(w, r) return } switch { + case snapshot && (isAPKIndex(rest) || isAPKSignature(rest)): + // A dated snapshot serves the kept version of the index; packages + // and other files below are the same as under the plain path. + h.serveSnapshot(w, r, repository, upstreamURL, rest, day) case isAPKIndex(rest) || isAPKSignature(rest): // Indexes and detached signatures are signed upstream metadata. // Cache them with the metadata TTL and serve the stored bytes @@ -138,7 +158,7 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques } downloadURL := upstreamURL + "/" + path - cacheFilename := repository + "/" + path + cacheFilename := APKCacheFilename(repository, path) h.proxy.Logger.Info("apk package download", "repository", repository, "name", name, "version", version, "arch", arch) @@ -160,8 +180,12 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques // metadata cache. Stored bytes are re-served verbatim, which keeps embedded // and detached signatures valid. func (h *APKHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, apkEcosystem, - h.metadataCacheKey(repository, upstreamURL, path), "*/*") + cacheKey := h.metadataCacheKey(repository, upstreamURL, path) + if h.snapshotsEnabled() { + h.serveMetadataWithSnapshot(w, r, upstreamURL+"/"+path, cacheKey) + return + } + h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, apkEcosystem, cacheKey, "*/*") } // metadataCacheKey derives the metadata cache key from the repository name, diff --git a/internal/handler/apk_snapshots.go b/internal/handler/apk_snapshots.go new file mode 100644 index 0000000..401dc00 --- /dev/null +++ b/internal/handler/apk_snapshots.go @@ -0,0 +1,239 @@ +package handler + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// apkSnapshotDateLayout is the date format of a snapshot path segment. +const apkSnapshotDateLayout = "2006-01-02" + +// WithSnapshots keeps earlier versions of the indexes this handler serves and +// makes them available under /apk/{name}@{YYYY-MM-DD}/, as signed upstream +// bytes that apk verifies as usual. keep is how long a version is kept after +// it was replaced; 0 disables snapshots. +func (h *APKHandler) WithSnapshots(keep time.Duration) *APKHandler { + h.snapshots = keep + return h +} + +// snapshotsEnabled reports whether snapshots are recorded and served. They +// need the metadata cache, which stores the documents they are copied from. +func (h *APKHandler) snapshotsEnabled() bool { + return h.snapshots > 0 && h.proxy.CacheMetadata && h.proxy.DB != nil && h.proxy.Storage != nil +} + +// resolveRepository maps the first path segment to a configured repository. +// A segment of the form {name}@{YYYY-MM-DD} names a snapshot of repository +// name for that day, when snapshots are enabled and no repository is +// configured under the full segment. ok is false for anything else. +func (h *APKHandler) resolveRepository(segment string) (repository, upstreamURL string, day time.Time, snapshot, ok bool) { + if upstreamURL, found := h.repositories[segment]; found { + return segment, upstreamURL, time.Time{}, false, true + } + if !h.snapshotsEnabled() { + return "", "", time.Time{}, false, false + } + at := strings.LastIndexByte(segment, '@') + if at < 0 { + return "", "", time.Time{}, false, false + } + upstreamURL, found := h.repositories[segment[:at]] + if !found { + return "", "", time.Time{}, false, false + } + day, err := time.Parse(apkSnapshotDateLayout, segment[at+1:]) + if err != nil { + return "", "", time.Time{}, false, false + } + return segment[:at], upstreamURL, day, true, true +} + +// serveSnapshot answers a request for an index or signature under a snapshot +// path with the version that was current at the end of day. A day for which +// no version was stored, or one that has not begun yet, is 404. +func (h *APKHandler) serveSnapshot(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, day time.Time) { + end := day.AddDate(0, 0, 1) + if day.After(time.Now()) { + http.NotFound(w, r) + return + } + snap, err := h.proxy.DB.MetadataSnapshotAt(apkEcosystem, h.metadataCacheKey(repository, upstreamURL, path), end) + if err != nil { + h.proxy.Logger.Error("apk snapshot lookup failed", "repository", repository, "path", path, "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + if snap == nil { + http.NotFound(w, r) + return + } + + if writeMetadataValidators(w, r, cachedMeta{etag: `"` + snap.ContentDigest + `"`, lastModified: snap.ValidFrom()}) { + return + } + body, err := h.proxy.Storage.Open(r.Context(), snap.StoragePath) + if err != nil { + h.proxy.Logger.Error("apk snapshot unreadable", "repository", repository, "path", path, + "storage_path", snap.StoragePath, "error", err) + http.Error(w, "snapshot unavailable", http.StatusInternalServerError) + return + } + defer func() { _ = body.Close() }() + + if snap.ContentType.String != "" { + w.Header().Set(headerContentType, snap.ContentType.String) + } + w.Header().Set(headerContentLength, strconv.FormatInt(snap.Size, 10)) + if snap.ContentEncoding.String != "" { + w.Header().Set(headerContentEncoding, snap.ContentEncoding.String) + } + w.WriteHeader(http.StatusOK) + if r.Method != http.MethodHead { + _, _ = io.Copy(w, body) + } +} + +// serveMetadataWithSnapshot is ProxyCached, recording each version it serves. +func (h *APKHandler) serveMetadataWithSnapshot(w http.ResponseWriter, r *http.Request, upstream, cacheKey string) { + ctx := context.WithoutCancel(r.Context()) + h.proxy.proxyCachedWithEncoding(w, r, upstream, apkEcosystem, cacheKey, "identity", &metadataHooks{ + hit: func(entry *database.MetadataCacheEntry, seen time.Time) { + h.recordStoredSnapshot(ctx, cacheKey, entry, seen) + }, + fetched: func(body []byte, contentType, contentEncoding string) { + h.recordServedSnapshot(ctx, cacheKey, body, contentType, contentEncoding, time.Now()) + }, + }, "*/*") +} + +// recordServedSnapshot keeps body, the document a request is serving, as the +// version current from observed, unless the version current then already has +// its bytes. +func (h *APKHandler) recordServedSnapshot(ctx context.Context, cacheKey string, body []byte, contentType, contentEncoding string, observed time.Time) { + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + if !h.snapshotNeeded(cacheKey, digest, observed) { + return + } + storagePath := apkSnapshotStoragePath(cacheKey, digest, observed) + if _, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(body)); err != nil { + h.proxy.Logger.Warn("failed to store apk snapshot", "path", storagePath, "error", err) + h.discardSnapshotBlob(ctx, storagePath) + return + } + h.addSnapshot(ctx, &database.MetadataSnapshot{ + Ecosystem: apkEcosystem, + Name: cacheKey, + StoragePath: storagePath, + ContentType: sql.NullString{String: contentType, Valid: contentType != ""}, + ContentEncoding: sql.NullString{String: contentEncoding, Valid: contentEncoding != ""}, + ContentDigest: digest, + Size: int64(len(body)), + ValidFromMS: observed.UnixMilli(), + }) +} + +// recordStoredSnapshot keeps the cached document entry describes as the +// version current from observed, unless the version current then already +// has its digest. +func (h *APKHandler) recordStoredSnapshot(ctx context.Context, cacheKey string, entry *database.MetadataCacheEntry, observed time.Time) { + if !entry.ContentDigest.Valid || !h.snapshotNeeded(cacheKey, entry.ContentDigest.String, observed) { + return + } + h.storeSnapshot(ctx, cacheKey, entry, observed) +} + +// snapshotNeeded reports whether a version with digest seen at observed +// differs from the version current then. It is only a shortcut that saves +// copying the document: AddMetadataSnapshot makes the same check. +func (h *APKHandler) snapshotNeeded(cacheKey, digest string, observed time.Time) bool { + current, err := h.proxy.DB.MetadataSnapshotAt(apkEcosystem, cacheKey, observed.Add(time.Millisecond)) + return err == nil && (current == nil || current.ContentDigest != digest) +} + +// storeSnapshot records the cached document entry as the version current +// from validFrom. Concurrent requests can both get here for the same +// version; each copies to a path of its own, so the one whose row is not +// added only deletes its own copy. +func (h *APKHandler) storeSnapshot(ctx context.Context, cacheKey string, entry *database.MetadataCacheEntry, validFrom time.Time) { + digest := entry.ContentDigest.String + storagePath := apkSnapshotStoragePath(cacheKey, digest, validFrom) + size, ok := h.copySnapshotBlob(ctx, entry.StoragePath, storagePath, digest) + if !ok { + return + } + h.addSnapshot(ctx, &database.MetadataSnapshot{ + Ecosystem: apkEcosystem, + Name: cacheKey, + StoragePath: storagePath, + ContentType: entry.ContentType, + ContentEncoding: entry.ContentEncoding, + ContentDigest: digest, + Size: size, + ValidFromMS: validFrom.UnixMilli(), + }) +} + +// addSnapshot adds the row for a snapshot whose blob is stored, and deletes +// the blob when the row isn't added, as when another request recorded the +// same version first. +func (h *APKHandler) addSnapshot(ctx context.Context, snap *database.MetadataSnapshot) { + added, err := h.proxy.DB.AddMetadataSnapshot(snap) + if err != nil { + h.proxy.Logger.Warn("failed to record apk snapshot", "key", snap.Name, "error", err) + } + if err != nil || !added { + h.discardSnapshotBlob(ctx, snap.StoragePath) + } +} + +// copySnapshotBlob copies the cached document at from to to and reports its +// size, provided the copied bytes still have digest. A concurrent fetch can +// replace the cached document after its row was read; that version is +// recorded by the request that stored it. +func (h *APKHandler) copySnapshotBlob(ctx context.Context, from, to, digest string) (int64, bool) { + src, err := h.proxy.Storage.Open(ctx, from) + if err != nil { + return 0, false + } + defer func() { _ = src.Close() }() + size, hash, err := h.proxy.Storage.Store(ctx, to, src) + if err != nil { + h.proxy.Logger.Warn("failed to store apk snapshot", "path", to, "error", err) + return 0, false + } + if "sha256:"+hash != digest { + h.discardSnapshotBlob(ctx, to) + return 0, false + } + return size, true +} + +func (h *APKHandler) discardSnapshotBlob(ctx context.Context, path string) { + if err := h.proxy.Storage.Delete(ctx, path); err != nil { + h.proxy.Logger.Warn("failed to discard apk snapshot blob", "path", path, "error", err) + } +} + +// apkSnapshotStoragePath returns a new path to store one kept version of the +// document cached under cacheKey at. The time it became current keeps a +// version that returns after an upgrade was rolled back apart from its +// earlier copy, and a random suffix keeps two requests recording the same +// version in the same millisecond apart. +func apkSnapshotStoragePath(cacheKey, digest string, validFrom time.Time) string { + return "_snapshots/" + apkEcosystem + "/" + cacheKey + "/" + + strconv.FormatInt(validFrom.UnixMilli(), 10) + "-" + strings.TrimPrefix(digest, "sha256:") + + "-" + rand.Text() +} diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go new file mode 100644 index 0000000..83bb26d --- /dev/null +++ b/internal/handler/apk_snapshots_test.go @@ -0,0 +1,382 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/registries/fetch" +) + +const apkSnapshotIndexPath = "/v3.22/main/x86_64/APKINDEX.tar.gz" + +// apkSnapshotUpstream serves the index held in index, initially "index v1", +// and counts package downloads. +type apkSnapshotUpstream struct { + index atomic.Value + downloads atomic.Int32 + server *httptest.Server +} + +func newAPKSnapshotUpstream(t *testing.T) *apkSnapshotUpstream { + t.Helper() + u := &apkSnapshotUpstream{} + u.index.Store("index v1") + u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case apkSnapshotIndexPath: + _, _ = w.Write([]byte(u.index.Load().(string))) + case "/v3.22/main/x86_64/curl-8.5.0-r0.apk": + u.downloads.Add(1) + _, _ = w.Write([]byte("curl 8.5.0")) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.server.Close) + return u +} + +// newSnapshotAPKHandler returns an APK handler with snapshots enabled whose +// metadata TTL is 0, so every index request asks upstream. +func newSnapshotAPKHandler(t *testing.T, upstreamURL string) (*APKHandler, *database.DB) { + t.Helper() + proxy, db, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 + proxy.HTTPClient = http.DefaultClient + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(http.DefaultClient), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewAPKHandler(proxy, "http://proxy.example", map[string]string{"alpine": upstreamURL}). + WithSnapshots(30 * 24 * time.Hour) + return h, db +} + +func snapshotCount(t *testing.T, db *database.DB) int { + t.Helper() + var n int + if err := db.Get(&n, `SELECT COUNT(*) FROM metadata_snapshots`); err != nil { + t.Fatal(err) + } + return n +} + +func expectAPKResponse(t *testing.T, h *APKHandler, target string, status int, body string) { + t.Helper() + w := serveAPKRequest(h, target) + if w.Code != status { + t.Fatalf("%s: status = %d, want %d: %s", target, w.Code, status, w.Body.String()) + } + if status == http.StatusOK && w.Body.String() != body { + t.Errorf("%s: body = %q, want %q", target, w.Body.String(), body) + } +} + +// TestAPKSnapshots_ServeIndexCurrentAtDate records two versions of an index +// through the plain path and checks that each dated path serves the version +// that was current at the end of that day. +func TestAPKSnapshots_ServeIndexCurrentAtDate(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, upstream.server.URL) + + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + // Pretend v1 was first stored on 2026-01-10. + recorded := time.Date(2026, 1, 10, 12, 0, 0, 0, time.UTC) + if _, err := db.Exec(`UPDATE metadata_snapshots SET valid_from_ms = ?`, recorded.UnixMilli()); err != nil { + t.Fatal(err) + } + + upstream.index.Store("index v2") + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v2") + // An unchanged index adds no snapshot. + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v2") + if got := snapshotCount(t, db); got != 2 { + t.Fatalf("snapshots = %d, want 2", got) + } + + today := time.Now().UTC().Format(apkSnapshotDateLayout) + future := time.Now().UTC().AddDate(0, 0, 2).Format(apkSnapshotDateLayout) + expectAPKResponse(t, h, "/alpine@2026-01-09"+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/alpine@2026-01-10"+apkSnapshotIndexPath, http.StatusOK, "index v1") + expectAPKResponse(t, h, "/alpine@"+today+apkSnapshotIndexPath, http.StatusOK, "index v2") + expectAPKResponse(t, h, "/alpine@"+future+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/alpine@2026-13-01"+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/other@2026-01-10"+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/alpine@2026-01-10/v3.22/community/x86_64/APKINDEX.tar.gz", http.StatusNotFound, "") +} + +func TestAPKSnapshots_ConditionalRequest(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, _ := newSnapshotAPKHandler(t, upstream.server.URL) + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + + target := "/alpine@" + time.Now().UTC().Format(apkSnapshotDateLayout) + apkSnapshotIndexPath + first := serveAPKRequest(h, target) + etag := first.Header().Get("ETag") + if first.Code != http.StatusOK || etag == "" { + t.Fatalf("status = %d, ETag = %q, want 200 with an ETag", first.Code, etag) + } + req := httptest.NewRequest(http.MethodGet, target, nil) + req.Header.Set("If-None-Match", etag) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusNotModified { + t.Errorf("conditional request: status = %d, want 304", w.Code) + } +} + +// TestAPKSnapshots_PackagesShareTheCache checks that a package requested +// under a snapshot path is the cached artifact of the plain path. +func TestAPKSnapshots_PackagesShareTheCache(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, _ := newSnapshotAPKHandler(t, upstream.server.URL) + + pkg := "/v3.22/main/x86_64/curl-8.5.0-r0.apk" + expectAPKResponse(t, h, "/alpine"+pkg, http.StatusOK, "curl 8.5.0") + expectAPKResponse(t, h, "/alpine@2026-01-10"+pkg, http.StatusOK, "curl 8.5.0") + if got := upstream.downloads.Load(); got != 1 { + t.Errorf("upstream downloads = %d, want 1", got) + } + expectAPKResponse(t, h, "/alpine@2026-01-10/v3.22/main/x86_64/gone-1.0-r0.apk", http.StatusNotFound, "") +} + +func TestAPKSnapshots_DisabledByDefault(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, upstream.server.URL) + h.WithSnapshots(0) + + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if got := snapshotCount(t, db); got != 0 { + t.Errorf("snapshots = %d, want 0", got) + } + today := time.Now().UTC().Format(apkSnapshotDateLayout) + expectAPKResponse(t, h, "/alpine@"+today+apkSnapshotIndexPath, http.StatusNotFound, "") +} + +// TestAPKSnapshots_ConfiguredNameWins checks that a repository whose +// configured name contains '@' is served as configured, not as a snapshot. +func TestAPKSnapshots_ConfiguredNameWins(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, _ := newSnapshotAPKHandler(t, upstream.server.URL) + h.repositories["alpine@2026-01-10"] = upstream.server.URL + + expectAPKResponse(t, h, "/alpine@2026-01-10"+apkSnapshotIndexPath, http.StatusOK, "index v1") +} + +// TestAPKSnapshots_SameVersionRecordedTwice checks that two requests +// recording the same version in the same millisecond, as coalesced fetches +// after an upstream update can, leave the recorded snapshot readable: the +// request whose row is not added must not delete the other's blob. +func TestAPKSnapshots_SameVersionRecordedTwice(t *testing.T) { + u := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, u.server.URL) + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + + key := h.metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) + entry, err := db.GetMetadataCache(apkEcosystem, key) + if err != nil || entry == nil { + t.Fatalf("cached index row = %v, %v", entry, err) + } + if _, err := db.Exec(`DELETE FROM metadata_snapshots`); err != nil { + t.Fatal(err) + } + validFrom := time.Now() + h.storeSnapshot(t.Context(), key, entry, validFrom) + h.storeSnapshot(t.Context(), key, entry, validFrom) + + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots recorded, want 1", n) + } + snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) + if err != nil || snap == nil { + t.Fatalf("latest snapshot = %v, %v", snap, err) + } + body, err := h.proxy.Storage.Open(t.Context(), snap.StoragePath) + if err != nil { + t.Fatalf("recorded snapshot blob unreadable: %v", err) + } + _ = body.Close() + day := validFrom.UTC().Format(apkSnapshotDateLayout) + expectAPKResponse(t, h, "/alpine@"+day+apkSnapshotIndexPath, http.StatusOK, "index v1") +} + +// pausingWriter blocks the first Write until release is closed, after +// signalling on writing. +type pausingWriter struct { + *httptest.ResponseRecorder + once sync.Once + writing chan struct{} + release chan struct{} +} + +func (w *pausingWriter) Write(b []byte) (int, error) { + w.once.Do(func() { + close(w.writing) + <-w.release + }) + return w.ResponseRecorder.Write(b) +} + +// TestAPKSnapshots_OverlappingRequests pauses a request while it writes +// index v1, lets a second request fetch and serve v2, and then lets the first +// one finish. Both versions must be kept, in the order they were fetched, +// with readable blobs. +func TestAPKSnapshots_OverlappingRequests(t *testing.T) { + u := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, u.server.URL) + target := "/alpine" + apkSnapshotIndexPath + + first := &pausingWriter{ + ResponseRecorder: httptest.NewRecorder(), + writing: make(chan struct{}), + release: make(chan struct{}), + } + done := make(chan struct{}) + go func() { + defer close(done) + h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, target, nil)) + }() + select { + case <-first.writing: + case <-done: + t.Fatal("first request finished without writing a body") + } + + u.index.Store("index v2") + expectAPKResponse(t, h, target, http.StatusOK, "index v2") + close(first.release) + <-done + if first.Code != http.StatusOK || first.Body.String() != "index v1" { + t.Fatalf("first request = %d %q, want 200 with index v1", first.Code, first.Body.String()) + } + + var snaps []database.MetadataSnapshot + if err := db.Select(&snaps, `SELECT `+ + `id, ecosystem, name, storage_path, content_type, content_encoding, content_digest, size, valid_from_ms `+ + `FROM metadata_snapshots ORDER BY valid_from_ms, id`); err != nil { + t.Fatal(err) + } + if len(snaps) != 2 { + t.Fatalf("%d snapshots recorded, want 2", len(snaps)) + } + for i, want := range []string{"index v1", "index v2"} { + body, err := h.proxy.Storage.Open(t.Context(), snaps[i].StoragePath) + if err != nil { + t.Fatalf("snapshot %d blob unreadable: %v", i, err) + } + got, _ := io.ReadAll(body) + _ = body.Close() + if string(got) != want { + t.Errorf("snapshot %d = %q, want %q", i, got, want) + } + if snaps[i].Size != int64(len(want)) { + t.Errorf("snapshot %d size = %d, want %d", i, snaps[i].Size, len(want)) + } + } +} + +// TestAPKSnapshots_ConcurrentReplicas records the same index version from +// two replicas sharing a Postgres database, many times at once. One snapshot +// row must remain, with a readable blob, and the other copies are deleted. +func TestAPKSnapshots_ConcurrentReplicas(t *testing.T) { + a, b, db := sharedInstances(t) + u := newAPKSnapshotUpstream(t) + handlers := make([]*APKHandler, 0, 2) + for _, p := range []*Proxy{a, b} { + p.CacheMetadata = true + p.HTTPClient = http.DefaultClient + handlers = append(handlers, NewAPKHandler(p, "http://proxy.example", map[string]string{"alpine": u.server.URL}). + WithSnapshots(30*24*time.Hour)) + } + expectAPKResponse(t, handlers[0], "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + + key := handlers[0].metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) + entry, err := db.GetMetadataCache(apkEcosystem, key) + if err != nil || entry == nil { + t.Fatalf("cached index row = %v, %v", entry, err) + } + if _, err := db.Exec(`DELETE FROM metadata_snapshots`); err != nil { + t.Fatal(err) + } + // The request above recorded a copy of its own, which the test leaves be. + store := a.Storage.(*mockStorage) + store.mu.Lock() + before := map[string]bool{} + for path := range store.files { + before[path] = true + } + store.mu.Unlock() + + validFrom := time.Now() + var wg sync.WaitGroup + for i := range 16 { + wg.Go(func() { handlers[i%2].storeSnapshot(t.Context(), key, entry, validFrom) }) + } + wg.Wait() + + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots recorded, want 1", n) + } + snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) + if err != nil || snap == nil { + t.Fatalf("latest snapshot = %v, %v", snap, err) + } + store.mu.Lock() + defer store.mu.Unlock() + if got := string(store.files[snap.StoragePath]); got != "index v1" { + t.Errorf("snapshot blob = %q, want the index", got) + } + for path := range store.files { + if strings.HasPrefix(path, "_snapshots/") && path != snap.StoragePath && !before[path] { + t.Errorf("copy %s of a snapshot that wasn't recorded is left in storage", path) + } + } +} + +// TestAPKSnapshots_CacheHit records an index cached before snapshots were +// enabled when a request is served from the cache within its TTL, and adds +// no row for a later hit on the same version. +func TestAPKSnapshots_CacheHit(t *testing.T) { + u := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, u.server.URL) + h.proxy.MetadataTTL = 5 * time.Minute + h.WithSnapshots(0) + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if n := snapshotCount(t, db); n != 0 { + t.Fatalf("%d snapshots recorded with snapshots off, want 0", n) + } + + h.WithSnapshots(30 * 24 * time.Hour) + // Upstream moved on; a hit still serves the cached v1. + u.index.Store("index v2") + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots recorded on a cache hit, want 1", n) + } + key := h.metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) + snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) + if err != nil || snap == nil { + t.Fatalf("latest snapshot = %v, %v", snap, err) + } + body, err := h.proxy.Storage.Open(t.Context(), snap.StoragePath) + if err != nil { + t.Fatalf("recorded snapshot blob unreadable: %v", err) + } + got, _ := io.ReadAll(body) + _ = body.Close() + if string(got) != "index v1" { + t.Fatalf("snapshot blob = %q, want %q", got, "index v1") + } + + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots after a second hit on the same version, want 1", n) + } +} diff --git a/internal/handler/arch.go b/internal/handler/arch.go index b53f25c..da1a6f1 100644 --- a/internal/handler/arch.go +++ b/internal/handler/arch.go @@ -199,12 +199,12 @@ func (h *ArchHandler) handleDatabase(w http.ResponseWriter, r *http.Request, rep d := h.database(repository, upstreamURL, path) if _, hit := p.cachedMetadataState(r.Context(), archEcosystem, d.dbKey, nil); hit != nil { metrics.RecordMetadataCacheHit(archEcosystem, len(hit.body)) - h.writeMetadata(w, r, d.dbKey, *hit) + h.writeMetadata(w, r, *hit) return } p.recordMetadataCacheMiss(archEcosystem) res := h.sharedRefresh(r.Context(), d, true) - h.writeMetadata(w, r, d.dbKey, res) + h.writeMetadata(w, r, res) } // handleDatabaseSignature serves the signature stored for the cached @@ -218,7 +218,7 @@ func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Req d := h.database(repository, upstreamURL, strings.TrimSuffix(path, archSignatureSuffix)) if res, ok := h.cachedSignature(r.Context(), d); ok { metrics.RecordMetadataCacheHit(archEcosystem, len(res.body)) - h.writeMetadata(w, r, d.sigKey, res) + h.writeMetadata(w, r, res) return } p.recordMetadataCacheMiss(archEcosystem) @@ -242,7 +242,7 @@ func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Req } return metadataResult{err: errArchSignatureUnpaired} }) - h.writeMetadata(w, r, d.sigKey, res) + h.writeMetadata(w, r, res) } var errArchSignatureUnpaired = errors.New("database signature could not be paired with the database") @@ -324,7 +324,7 @@ func (h *ArchHandler) fetchDatabase(ctx context.Context, d archDatabase, entry * return metadataResult{err: fmt.Errorf("upstream failed and no cached metadata: %w", err)}, nil } p.Logger.Warn("upstream metadata fetch failed, checking cache", "ecosystem", archEcosystem, "key", d.dbKey, "error", err) - return h.readDatabase(ctx, p.currentMetadataEntry(archEcosystem, d.dbKey, entry), err) + return h.readDatabase(ctx, p.currentMetadataEntry(archEcosystem, d.dbKey, entry), err, true) } // storeDatabase caches meta under its digest and returns the database the @@ -345,15 +345,15 @@ func (h *ArchHandler) storeDatabase(ctx context.Context, d archDatabase, meta *u row, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) if row == nil { // Nothing cached at all: serve it like an uncached proxy would. - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding}, nil + return fetchedResult(meta, ""), nil } if row.StoragePath != path { - return h.readDatabase(ctx, row, errArchDatabaseNotCached) + return h.readDatabase(ctx, row, errArchDatabaseNotCached, false) } if previous != nil && previous.StoragePath != path { _ = p.Storage.Delete(ctx, previous.StoragePath) } - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding}, row + return fetchedResult(meta, row.ContentDigest.String), row } // blobExists reports whether path is stored. Size eviction can remove a blob @@ -377,13 +377,18 @@ func (h *ArchHandler) touchDatabase(row *database.MetadataCacheEntry, meta *upst var errArchDatabaseNotCached = errors.New("database could not be cached") // readDatabase returns the cached database row's bytes, or cause if they are -// unreadable. -func (h *ArchHandler) readDatabase(ctx context.Context, row *database.MetadataCacheEntry, cause error) (metadataResult, *database.MetadataCacheEntry) { +// unreadable. stale marks them as served because the upstream failed with +// cause. +func (h *ArchHandler) readDatabase(ctx context.Context, row *database.MetadataCacheEntry, cause error, stale bool) (metadataResult, *database.MetadataCacheEntry) { body, contentType, err := h.proxy.readCachedMetadata(ctx, row, nil) if err != nil { return metadataResult{err: fmt.Errorf("cached database unusable (%v): %w", err, cause)}, nil } - return metadataResult{body: body, contentType: contentType, contentEncoding: row.ContentEncoding.String}, row + res := metadataResult{body: body, contentType: contentType, contentEncoding: row.ContentEncoding.String} + if stale { + res.stale, res.upstreamErr = true, cause + } + return res.describedBy(row), row } // lock serializes cache writes for one database and its signature, so a row @@ -433,7 +438,7 @@ func (h *ArchHandler) storedSignature(ctx context.Context, d archDatabase, dbDig if err != nil { return metadataResult{}, false } - return metadataResult{body: body, contentType: contentType, contentEncoding: entry.ContentEncoding.String}, true + return metadataResult{body: body, contentType: contentType, contentEncoding: entry.ContentEncoding.String}.describedBy(entry), true } return metadataResult{}, false } @@ -478,7 +483,7 @@ func archDigest(body []byte) string { return "sha256:" + hex.EncodeToString(digest[:]) } -func (h *ArchHandler) writeMetadata(w http.ResponseWriter, r *http.Request, cacheKey string, res metadataResult) { +func (h *ArchHandler) writeMetadata(w http.ResponseWriter, r *http.Request, res metadataResult) { if res.err != nil { if errors.Is(res.err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) @@ -491,7 +496,7 @@ func (h *ArchHandler) writeMetadata(w http.ResponseWriter, r *http.Request, cach http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } - h.proxy.writeMetadataCachedResponseWithEncoding(w, r, archEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) + writeMetadataResult(w, r, res) } // metadataCacheKey hashes repository, upstream URL and path, as APKHandler does. diff --git a/internal/handler/arch_test.go b/internal/handler/arch_test.go index 3affad5..02c87fa 100644 --- a/internal/handler/arch_test.go +++ b/internal/handler/arch_test.go @@ -1,6 +1,7 @@ package handler import ( + "database/sql" "fmt" "net/http" "net/http/httptest" @@ -648,3 +649,55 @@ func serveArchRequest(h *ArchHandler, method, target string, header http.Header) h.Routes().ServeHTTP(w, req) return w } + +// TestArchHandler_DatabaseValidatorsDescribeBody checks a fetched database +// goes out with its own Last-Modified, not that of the cache row, which +// another instance may have rewritten while the signature was fetched. +func TestArchHandler_DatabaseValidatorsDescribeBody(t *testing.T) { + const dbPath, sigPath = "/core/os/x86_64/core.db", "/core/os/x86_64/core.db.sig" + fetched := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + rewritten := fetched.Add(time.Hour) + upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("core-db-v1"), sigPath: []byte("sig")}) + upstream.modified[dbPath] = fetched + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + key := h.metadataCacheKey("archlinux", upstream.URL, "core/os/x86_64/core.db") + upstream.served = func(path string) { + if path != sigPath { + return + } + entry, err := proxy.DB.GetMetadataCache(archEcosystem, key) + if err != nil || entry == nil { + t.Errorf("database row while fetching the signature: %v", err) + return + } + entry.LastModified = sql.NullTime{Time: rewritten, Valid: true} + if err := proxy.DB.UpsertMetadataCache(entry); err != nil { + t.Error(err) + } + } + + w := serveArchRequest(h, http.MethodGet, "/archlinux"+dbPath, nil) + assertArchBody(t, w, "database", []byte("core-db-v1")) + if got := w.Header().Get("Last-Modified"); got != fetched.Format(http.TimeFormat) { + t.Errorf("Last-Modified = %q, want the fetched database's %q", got, fetched.Format(http.TimeFormat)) + } +} + +// TestArchHandler_StaleDatabaseMarkedWithoutTTL checks the stale fallback +// carries the Warning header with metadata_ttl 0. +func TestArchHandler_StaleDatabaseMarkedWithoutTTL(t *testing.T) { + db := []byte("core-db-bytes") + upstream := newArchUpstream(t, map[string][]byte{"/core/os/x86_64/core.db": db}) + proxy := newArchTestProxy(t, upstream) + proxy.MetadataTTL = 0 + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil), "seed", db) + upstream.available.Store(false) + w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil) + assertArchBody(t, w, "stale", db) + if got := w.Header().Get("Warning"); got != staleWarning { + t.Errorf("Warning = %q, want %q", got, staleWarning) + } +} diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 0c26551..0c74214 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -2,6 +2,7 @@ package handler import ( "bufio" + "bytes" "encoding/json" "errors" "fmt" @@ -27,7 +28,11 @@ const ( // they are, so package rules use the default key mapping and have to match // that name exactly. func init() { - retention.Default.Register(retention.Spec{Key: "cargo", CanonicalPackage: retention.DefaultCanonical("cargo")}) + retention.Default.Register(retention.Spec{ + Key: "cargo", + CanonicalPackage: retention.DefaultCanonical("cargo"), + CompareVersions: retention.VersionOrder("cargo"), + }) } // CargoHandler handles cargo registry protocol requests. @@ -186,6 +191,32 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr } } +// publishTime reads a crate version's pubtime from the sparse index, sharing +// the index cache with handleIndex. Entries published before crates.io +// recorded pubtime have none. +func (h *CargoHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + upstreamURL := fmt.Sprintf("%s/%s", h.indexURL, h.buildIndexPath(name)) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "cargo", name, upstreamURL, "text/plain") + if err != nil { + return time.Time{}, false, err + } + for line := range bytes.SplitSeq(body, []byte("\n")) { + if len(bytes.TrimSpace(line)) == 0 { + continue + } + var entry crateIndexEntry + if err := json.Unmarshal(line, &entry); err != nil { + return time.Time{}, false, err + } + if entry.Version != version { + continue + } + published, err := time.Parse(time.RFC3339, entry.PublishTime) + return published, err == nil, nil + } + return time.Time{}, false, nil +} + // buildIndexPath builds the sparse index path for a crate name. func (h *CargoHandler) buildIndexPath(name string) string { name = strings.ToLower(name) @@ -217,6 +248,16 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("cargo download request", "crate", name, "version", version, "filename", filename) + if h.proxy.withholdInCooldown(w, r, downloadVersion{ + ecosystem: "cargo", policyName: name, storedName: name, version: version, cacheName: name, filename: filename, + lookup: func() (time.Time, bool, bool, error) { + published, ok, err := h.publishTime(r, name, version) + return published, ok, false, err + }, + }) { + return + } + downloadURL := fmt.Sprintf( "%s/%s/%s", h.downloadURL, diff --git a/internal/handler/composer.go b/internal/handler/composer.go index ccb42bb..6b82cff 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -55,6 +55,7 @@ type ComposerHandler struct { repoURL string proxyURL string routes packageRoutes + indexCache composerIndexCache } // NewComposerHandler creates a new Composer protocol handler. @@ -100,12 +101,20 @@ func (h *ComposerHandler) Routes() http.Handler { mux.HandleFunc("GET /search.json", h.proxyUpstream) mux.HandleFunc("GET /packages/list.json", h.proxyUpstream) + // Security advisories and filter lists, as the default repository offers them + mux.HandleFunc("GET "+composerAdvisoriesPath, h.handleSecurityAdvisories) + mux.HandleFunc("POST "+composerAdvisoriesPath, h.handleSecurityAdvisories) + mux.HandleFunc("GET "+composerFilterSummaryPath, h.handleFilterSummary) + return mux } -// handleServiceIndex returns the Composer repository service index. +// handleServiceIndex returns the Composer repository service index. It +// carries the security advisories and filter list entries of the default +// repository's own index when that has them, kept in memory for the metadata +// TTL; if the repository has not been reached, the index goes out without +// them. func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request) { - // Return a minimal service index pointing to our proxy index := map[string]any{ "packages": map[string]any{}, "metadata-url": h.proxyURL + "/composer/p2/%package%.json", @@ -113,6 +122,9 @@ func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Requ "search": h.proxyURL + "/composer/search.json?q=%query%&type=%type%", "providers-lazy-url": h.proxyURL + "/composer/p2/%package%.json", } + if upstream, ok := h.upstreamIndex(r.Context()); ok { + upstream.advertise(index, h.proxyURL) + } w.Header().Set(headerContentType, "application/json") _ = json.NewEncoder(w).Encode(index) @@ -419,8 +431,9 @@ func composerPublishTime(published []byte) (time.Time, bool) { // the package name to it. Composer only falls back to notify-batch when a // version has no notification-url of its own, and skips an empty one. Setting // it on the expanded fields keeps it set for the versions that inherit them. +// Composer reads a null notification-url as absent too. func disableDefaultNotification(fields *composerFields) { - if fields.get("notification-url") == nil { + if v := fields.get("notification-url"); v == nil || string(v) == "null" { fields.set("notification-url", []byte(`"notification-url"`), []byte(`""`)) } } @@ -905,9 +918,9 @@ type composerDist struct { } // findDownloadURLFromMetadata fetches a metadata document and returns the dist -// of the given version, with an empty URL if the version is not present. An -// error is returned only on transport failure; a missing document (non-200) -// or a missing version both yield an empty URL so the caller can fall back. +// of the given version, with an empty URL if the version is not present. A +// missing document (404 or 410) or a missing version both yield an empty URL +// so the caller can fall back; any other failure is an error. func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (composerDist, error) { h.proxy.Logger.Debug("fetching upstream metadata for download lookup", "url", metaURL, "package", packageName, "version", version) @@ -926,8 +939,16 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU h.proxy.Logger.Debug("upstream metadata response", "url", metaURL, "status", resp.StatusCode) - if resp.StatusCode != http.StatusOK { + switch { + case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: return composerDist{}, nil + case resp.StatusCode == http.StatusTooManyRequests && h.proxy.RateLimits != nil: + if err := h.proxy.RateLimits.Held(metaURL); err != nil { + return composerDist{}, err + } + } + if resp.StatusCode != http.StatusOK { + return composerDist{}, fmt.Errorf("upstream metadata returned %d", resp.StatusCode) } body, err := h.proxy.ReadMetadata(resp.Body) diff --git a/internal/handler/composer_advisories.go b/internal/handler/composer_advisories.go new file mode 100644 index 0000000..aa0fa26 --- /dev/null +++ b/internal/handler/composer_advisories.go @@ -0,0 +1,351 @@ +package handler + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "golang.org/x/sync/singleflight" +) + +const ( + composerAdvisoriesPath = "/api/security-advisories/" + composerFilterSummaryPath = "/lists/summary.json" + composerPackagesParam = "packages" + + // composerAdvisoriesMaxBody caps an advisories query. Composer sends at + // most 500 package names per request. + composerAdvisoriesMaxBody = 1 << 20 +) + +// composerUpstreamIndex holds the parts of the default repository's +// packages.json that the proxy advertises in its own: the security +// advisories API and the filter lists (Packagist's malware list). Composer +// reads advisories and filter entries for each package from its p2 metadata, +// which the proxy passes on unchanged, and asks these URLs for the rest. +type composerUpstreamIndex struct { + advisoriesURL string + advisoriesMetadata bool + filterLists json.RawMessage + filterSummaryURL string +} + +// composerIndexFallbackTTL is how long the upstream index is kept when the +// metadata TTL is 0. +const composerIndexFallbackTTL = time.Hour + +// composerIndexCache keeps the default repository's parsed packages.json in +// memory, whether or not metadata caching is on, so that serving the proxy's +// own packages.json does not wait on the upstream. A failed fetch is +// remembered for the same interval: the last good copy, or none, is used +// until then instead of retrying on every request. The zero value is ready +// to use. +type composerIndexCache struct { + mu sync.Mutex + index composerUpstreamIndex + ok bool + fetchedAt time.Time + group singleflight.Group + now func() time.Time // for tests; nil means time.Now +} + +func (c *composerIndexCache) clock() time.Time { + if c.now != nil { + return c.now() + } + return time.Now() +} + +// upstreamIndex returns the default repository's packages.json fields. ok is +// false when the repository has not been reached since the proxy started; a +// document without the fields yields an empty index. +func (h *ComposerHandler) upstreamIndex(ctx context.Context) (composerUpstreamIndex, bool) { + c := &h.indexCache + ttl := h.proxy.MetadataTTL + if ttl <= 0 { + ttl = composerIndexFallbackTTL + } + c.mu.Lock() + if !c.fetchedAt.IsZero() && c.clock().Sub(c.fetchedAt) < ttl { + index, ok := c.index, c.ok + c.mu.Unlock() + return index, ok + } + c.mu.Unlock() + + // One fetch serves every waiting request. It is detached from the + // request that started it, so a client hanging up neither fails the + // others nor gets remembered as an upstream failure. + _, _, _ = c.group.Do("index", func() (any, error) { + index, err := h.fetchUpstreamIndex(context.WithoutCancel(ctx)) + c.mu.Lock() + defer c.mu.Unlock() + if err == nil { + c.index, c.ok = index, true + } + c.fetchedAt = c.clock() + return nil, nil + }) + c.mu.Lock() + defer c.mu.Unlock() + return c.index, c.ok +} + +// fetchUpstreamIndex fetches and parses the default repository's +// packages.json. A document that doesn't parse is rejected before it is +// cached, so a broken response never replaces a good copy: with metadata +// caching the stored copy is used instead, and without it the error keeps +// the copy in memory. +func (h *ComposerHandler) fetchUpstreamIndex(ctx context.Context) (composerUpstreamIndex, error) { + indexURL := h.repoURL + "/packages.json" + cacheKey := hashedCacheKey(h.repoURL, "/packages.json") + validate := func(body []byte) error { + _, err := parseComposerUpstreamIndex(indexURL, body) + return err + } + res := h.proxy.fetchMetadataDocument(ctx, "composer", cacheKey, indexURL, contentTypeJSON, validate) + if res.err != nil { + h.proxy.Logger.Warn("failed to fetch upstream composer packages.json", "url", indexURL, "error", res.err) + return composerUpstreamIndex{}, res.err + } + return parseComposerUpstreamIndex(indexURL, res.body) +} + +// errComposerIndexNotObject is returned for a packages.json that is valid +// JSON but not an object, such as null. +var errComposerIndexNotObject = errors.New("composer packages.json is not a JSON object") + +// parseComposerUpstreamIndex reads the advisories and filter fields of a +// packages.json. A document without them yields an empty index; one that is +// not a JSON object, or has them in the wrong shape, is an error. +func parseComposerUpstreamIndex(indexURL string, body []byte) (composerUpstreamIndex, error) { + var doc struct { + SecurityAdvisories struct { + Metadata bool `json:"metadata"` + APIURL string `json:"api-url"` + } `json:"security-advisories"` + Filter struct { + Metadata bool `json:"metadata"` + Lists json.RawMessage `json:"lists"` + SummaryURL string `json:"summary-url"` + } `json:"filter"` + } + var index composerUpstreamIndex + if trimmed := bytes.TrimSpace(body); len(trimmed) == 0 || trimmed[0] != '{' { + return index, errComposerIndexNotObject + } + if err := json.Unmarshal(body, &doc); err != nil { + return index, fmt.Errorf("parsing composer packages.json: %w", err) + } + // Composer refuses advisories configured without an api-url unless the + // repository lists its packages, which the proxy does not. + if apiURL, ok := resolveComposerURL(indexURL, doc.SecurityAdvisories.APIURL); ok { + index.advisoriesURL = apiURL + index.advisoriesMetadata = doc.SecurityAdvisories.Metadata + } + // Composer only applies filter lists a repository marks with metadata. + // A filter api-url is not passed on: without it Composer reads the + // entries from package metadata, narrowed by the summary. + if doc.Filter.Metadata && len(doc.Filter.Lists) > 0 && doc.Filter.Lists[0] == '{' { + index.filterLists = doc.Filter.Lists + index.filterSummaryURL, _ = resolveComposerURL(indexURL, doc.Filter.SummaryURL) + } + return index, nil +} + +// resolveComposerURL resolves a URL from packages.json against the document's +// own, as Composer does, and accepts only http and https. +func resolveComposerURL(base, ref string) (string, bool) { + if ref == "" { + return "", false + } + b, err := url.Parse(base) + if err != nil { + return "", false + } + u, err := b.Parse(ref) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return "", false + } + return u.String(), true +} + +// advertise adds the advisories and filter entries to the proxy's +// packages.json, pointing at the proxy. +func (index composerUpstreamIndex) advertise(packages map[string]any, proxyURL string) { + if index.advisoriesURL != "" { + packages["security-advisories"] = map[string]any{ + "metadata": index.advisoriesMetadata, + "api-url": proxyURL + "/composer" + composerAdvisoriesPath, + } + } + if index.filterLists != nil { + filter := map[string]any{"metadata": true, "lists": index.filterLists} + if index.filterSummaryURL != "" { + filter["summary-url"] = proxyURL + "/composer" + composerFilterSummaryPath + } + packages["filter"] = filter + } +} + +// handleSecurityAdvisories relays a security advisories query to the URL the +// default repository advertises. Composer sends the package names as +// packages[] in a POST form; a GET query is accepted too. Names served by a +// package route or a hosted-only vendor are removed first, so private package +// names are never sent to the public API. +// +// Responses are not cached: each query names a different set of packages. +func (h *ComposerHandler) handleSecurityAdvisories(w http.ResponseWriter, r *http.Request) { + query, err := h.advisoriesQuery(w, r) + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + JSONError(w, http.StatusRequestEntityTooLarge, "advisories request too large") + return + } + JSONError(w, http.StatusBadRequest, "could not read advisories request") + return + } + + asked := len(query[composerPackagesParam+"[]"]) + public := h.publicPackageNames(query[composerPackagesParam+"[]"]) + if asked > 0 && len(public) == 0 { + w.Header().Set(headerContentType, contentTypeJSON) + _, _ = io.WriteString(w, `{"advisories":{}}`) + return + } + if asked > 0 { + query[composerPackagesParam+"[]"] = public + } + + index, ok := h.upstreamIndex(r.Context()) + if !ok { + JSONError(w, http.StatusBadGateway, "upstream repository unavailable") + return + } + if index.advisoriesURL == "" { + JSONError(w, http.StatusNotFound, "upstream repository has no security advisories API") + return + } + + req, err := h.advisoriesRequest(r, index.advisoriesURL, query) + if err != nil { + JSONError(w, http.StatusInternalServerError, "failed to create request") + return + } + h.proxy.Logger.Debug("composer advisories request", "packages", len(public), "withheld", asked-len(public)) + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("composer advisories request failed", "error", err) + JSONError(w, http.StatusBadGateway, "failed to reach upstream repository") + return + } + defer func() { _ = resp.Body.Close() }() + + h.proxy.relayResponse(w, r, resp, nil) +} + +// advisoriesQuery reads the query of an advisories request, from the form +// body of a POST or the URL of a GET. Package names come back under +// "packages[]" whichever spelling the client used: PHP's http_build_query, +// which Composer uses, numbers them as packages[0], packages[1], ... +func (h *ComposerHandler) advisoriesQuery(w http.ResponseWriter, r *http.Request) (url.Values, error) { + raw := r.URL.RawQuery + if r.Method == http.MethodPost { + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, composerAdvisoriesMaxBody)) + if err != nil { + return nil, err + } + raw = string(body) + } + values, err := url.ParseQuery(raw) + if err != nil { + return nil, err + } + query := url.Values{} + for key, vals := range values { + if key == composerPackagesParam || strings.HasPrefix(key, composerPackagesParam+"[") { + key = composerPackagesParam + "[]" + } + query[key] = append(query[key], vals...) + } + return query, nil +} + +// publicPackageNames drops the names the default repository does not serve. +func (h *ComposerHandler) publicPackageNames(names []string) []string { + public := make([]string, 0, len(names)) + for _, name := range names { + if _, routed := h.routes.match(name); routed || h.proxy.hostedOnly("composer", name) { + continue + } + public = append(public, name) + } + return public +} + +// advisoriesRequest builds the upstream request, sending the query the way +// the client did. +func (h *ComposerHandler) advisoriesRequest(r *http.Request, apiURL string, query url.Values) (*http.Request, error) { + u, err := url.Parse(apiURL) + if err != nil { + return nil, err + } + var body io.Reader + if r.Method == http.MethodPost { + body = strings.NewReader(query.Encode()) + } else { + u.RawQuery = joinQuery(u.RawQuery, query.Encode()) + } + req, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), body) + if err != nil { + return nil, err + } + if r.Method == http.MethodPost { + req.Header.Set(headerContentType, "application/x-www-form-urlencoded") + } + for _, header := range []string{headerAccept, headerAcceptEncoding} { + if v := r.Header.Get(header); v != "" { + req.Header.Set(header, v) + } + } + return req, nil +} + +func joinQuery(a, b string) string { + if a == "" || b == "" { + return a + b + } + return a + "&" + b +} + +// handleFilterSummary relays the filter list summary the default repository +// advertises. It lists every package on a filter list, so the request names +// no packages. It is not cached, so a newly listed package is blocked as soon +// as the upstream lists it; Composer revalidates its own copy with +// If-Modified-Since, which is passed on. +func (h *ComposerHandler) handleFilterSummary(w http.ResponseWriter, r *http.Request) { + index, ok := h.upstreamIndex(r.Context()) + if !ok { + JSONError(w, http.StatusBadGateway, "upstream repository unavailable") + return + } + if index.filterSummaryURL == "" { + JSONError(w, http.StatusNotFound, "upstream repository has no filter list summary") + return + } + h.proxy.ProxyUpstream(w, r, index.filterSummaryURL, + []string{headerAccept, headerAcceptEncoding, "If-Modified-Since", "If-None-Match"}) +} diff --git a/internal/handler/composer_advisories_test.go b/internal/handler/composer_advisories_test.go new file mode 100644 index 0000000..097d8cc --- /dev/null +++ b/internal/handler/composer_advisories_test.go @@ -0,0 +1,470 @@ +package handler + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// composerAdvisoryUpstream serves a packages.json advertising security +// advisories and a malware filter list, and records the advisories and +// summary requests it gets. +type composerAdvisoryUpstream struct { + *httptest.Server + index string + down atomic.Bool // answer every request with 503 + // malformed answers packages.json with an HTML page and status 200. + malformed atomic.Bool + + mu sync.Mutex + requests []*recordedRequest +} + +type recordedRequest struct { + method string + path string + query url.Values + body string + header http.Header +} + +const composerAdvisoriesResponse = `{"advisories":{"public/library":[{"advisoryId":"PKSA-1","affectedVersions":"<1.0.1"}]}}` + +const composerFilterSummary = `{"filter":{"malware":{"evil/package":"1.0.0"}}}` + +func newComposerAdvisoryUpstream(t *testing.T) *composerAdvisoryUpstream { + t.Helper() + u := &composerAdvisoryUpstream{} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + u.mu.Lock() + u.requests = append(u.requests, &recordedRequest{r.Method, r.URL.Path, r.URL.Query(), string(body), r.Header.Clone()}) + u.mu.Unlock() + if u.down.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/packages.json": + if u.malformed.Load() { + w.Header().Set("Content-Type", "text/html") + _, _ = io.WriteString(w, "Service temporarily unavailable") + return + } + _, _ = io.WriteString(w, u.index) + case "/api/security-advisories/": + _, _ = io.WriteString(w, composerAdvisoriesResponse) + case "/lists/all/summary.json": + if r.Header.Get("If-Modified-Since") != "" { + w.WriteHeader(http.StatusNotModified) + return + } + _, _ = io.WriteString(w, composerFilterSummary) + default: + http.NotFound(w, r) + } + })) + // Like Packagist's: the API on one host, the summary as a relative URL. + u.index = `{ + "packages": [], + "metadata-url": "/p2/%package%.json", + "security-advisories": {"metadata": true, "api-url": "` + u.URL + `/api/security-advisories/"}, + "filter": {"metadata": true, "lists": {"malware": {"enabled": true}}, "summary-url": "/lists/all/summary.json"} + }` + t.Cleanup(u.Close) + return u +} + +// requestsTo returns the recorded requests for path. +func (u *composerAdvisoryUpstream) requestsTo(path string) []*recordedRequest { + u.mu.Lock() + defer u.mu.Unlock() + var out []*recordedRequest + for _, r := range u.requests { + if r.path == path { + out = append(out, r) + } + } + return out +} + +func newAdvisoryComposerHandler(t *testing.T, upstream *composerAdvisoryUpstream) *ComposerHandler { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HostedEnabled = true + proxy.HostedComposerVendors = []string{"acme"} + return NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL). + WithPackageRoutes(map[string]string{"example/*": "https://packages.example.com/composer"}) +} + +func serveComposerIndex(t *testing.T, h *ComposerHandler) map[string]any { + t.Helper() + w := serveComposer(h, "/packages.json") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body) + } + var index map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &index); err != nil { + t.Fatalf("parsing packages.json: %v", err) + } + return index +} + +func TestComposerServiceIndexAdvertisesAdvisoriesAndFilter(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + index := serveComposerIndex(t, newAdvisoryComposerHandler(t, upstream)) + + advisories, _ := index["security-advisories"].(map[string]any) + if advisories["api-url"] != "http://proxy.test/composer/api/security-advisories/" || advisories["metadata"] != true { + t.Errorf("security-advisories = %v, want the proxy's api-url with metadata", index["security-advisories"]) + } + filter, _ := index["filter"].(map[string]any) + if filter["summary-url"] != "http://proxy.test/composer/lists/summary.json" || filter["metadata"] != true { + t.Errorf("filter = %v, want the proxy's summary-url with metadata", index["filter"]) + } + lists, _ := filter["lists"].(map[string]any) + if _, ok := lists["malware"]; !ok { + t.Errorf("filter lists = %v, want upstream's malware list", filter["lists"]) + } + if index["metadata-url"] != "http://proxy.test/composer/p2/%package%.json" { + t.Errorf("metadata-url = %v, want the proxy's", index["metadata-url"]) + } +} + +func TestComposerServiceIndexWithoutUpstreamSupport(t *testing.T) { + tests := []struct { + name string + index string + }{ + {"not advertised", `{"packages": [], "metadata-url": "/p2/%package%.json"}`}, + {"advisories without api-url", `{"security-advisories": {"metadata": true}}`}, + {"filter without metadata", `{"filter": {"lists": {"malware": {"enabled": true}}, "api-url": "/filter"}}`}, + {"not json", ``}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + upstream.index = tt.index + index := serveComposerIndex(t, newAdvisoryComposerHandler(t, upstream)) + for _, key := range []string{"security-advisories", "filter"} { + if _, ok := index[key]; ok { + t.Errorf("%s advertised although upstream does not offer it: %v", key, index[key]) + } + } + }) + } +} + +func TestComposerServiceIndexUpstreamDown(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + upstream.Close() + + index := serveComposerIndex(t, h) + if _, ok := index["security-advisories"]; ok { + t.Errorf("security-advisories advertised without an upstream index: %v", index["security-advisories"]) + } + if index["metadata-url"] == nil { + t.Error("packages.json lacks metadata-url while the upstream is down") + } +} + +func advertisesAdvisories(index map[string]any) bool { + _, ok := index["security-advisories"] + return ok +} + +func TestComposerServiceIndexUpstreamDownNotRetried(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + upstream.down.Store(true) + h := newAdvisoryComposerHandler(t, upstream) + + serveComposerIndex(t, h) + fetched := len(upstream.requestsTo("/packages.json")) + if fetched == 0 { + t.Fatal("first packages.json made no upstream request") + } + index := serveComposerIndex(t, h) + if got := len(upstream.requestsTo("/packages.json")); got != fetched { + t.Errorf("second packages.json made %d upstream requests, want none", got-fetched) + } + if advertisesAdvisories(index) { + t.Errorf("security-advisories advertised without an upstream index: %v", index["security-advisories"]) + } +} + +func TestComposerServiceIndexKeptInMemory(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + h.proxy.MetadataTTL = 0 + now := time.Now() + h.indexCache.now = func() time.Time { return now } + + if !advertisesAdvisories(serveComposerIndex(t, h)) { + t.Fatal("security-advisories not advertised") + } + if got := len(upstream.requestsTo("/packages.json")); got != 1 { + t.Fatalf("upstream packages.json requests = %d, want 1", got) + } + + // Within the interval the copy in memory is served without asking. + upstream.down.Store(true) + now = now.Add(composerIndexFallbackTTL - time.Minute) + if !advertisesAdvisories(serveComposerIndex(t, h)) { + t.Error("security-advisories dropped within the interval") + } + if got := len(upstream.requestsTo("/packages.json")); got != 1 { + t.Errorf("upstream packages.json requests = %d within the interval, want 1", got) + } + + // After it the index is fetched again; the failure keeps the last good + // copy and is not retried until the next interval. + now = now.Add(2 * time.Minute) + if !advertisesAdvisories(serveComposerIndex(t, h)) { + t.Error("last good copy dropped after a failed fetch") + } + fetched := len(upstream.requestsTo("/packages.json")) + if fetched < 2 { + t.Fatalf("upstream packages.json requests = %d after the interval, want a new fetch", fetched) + } + serveComposerIndex(t, h) + if got := len(upstream.requestsTo("/packages.json")); got != fetched { + t.Errorf("failed fetch retried %d times within the interval", got-fetched) + } +} + +// assertComposerSecurityAdvertised checks that h advertises the upstream's +// advisories API and malware filter, and relays both. +func assertComposerSecurityAdvertised(t *testing.T, h *ComposerHandler, when string) { + t.Helper() + index := serveComposerIndex(t, h) + if !advertisesAdvisories(index) { + t.Errorf("%s: security-advisories not advertised: %v", when, index) + } + if _, ok := index["filter"]; !ok { + t.Errorf("%s: filter not advertised: %v", when, index) + } + w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B%5D=public%2Flibrary") + if w.Code != http.StatusOK || w.Body.String() != composerAdvisoriesResponse { + t.Errorf("%s: advisories = %d %q, want upstream's advisories", when, w.Code, w.Body) + } + if w := serveComposer(h, "/lists/summary.json"); w.Code != http.StatusOK || w.Body.String() != composerFilterSummary { + t.Errorf("%s: filter summary = %d %q, want upstream's summary", when, w.Code, w.Body) + } +} + +// A packages.json that doesn't parse, sent with status 200, must not replace +// the last good index, with metadata caching on or off: the advisories API +// and the malware filter stay advertised and relayed. +func TestComposerServiceIndexMalformedRefreshKeepsLastGood(t *testing.T) { + for _, cacheMetadata := range []bool{false, true} { + t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + h.proxy.CacheMetadata = cacheMetadata + h.proxy.MetadataTTL = time.Hour + now := time.Now() + h.indexCache.now = func() time.Time { return now } + cacheKey := hashedCacheKey(upstream.URL, "/packages.json") + + assertComposerSecurityAdvertised(t, h, "before the refresh") + var stored *database.MetadataCacheEntry + if cacheMetadata { + var err error + if stored, err = h.proxy.DB.GetMetadataCache("composer", cacheKey); err != nil || stored == nil { + t.Fatalf("packages.json not cached: %v, %v", stored, err) + } + expireMetadata(t, h.proxy.DB, "composer", cacheKey) + } + + upstream.malformed.Store(true) + now = now.Add(2 * time.Hour) + fetched := len(upstream.requestsTo("/packages.json")) + assertComposerSecurityAdvertised(t, h, "after a malformed refresh") + if got := len(upstream.requestsTo("/packages.json")); got == fetched { + t.Fatal("packages.json was not fetched again after the TTL") + } + + if !cacheMetadata { + return + } + after, err := h.proxy.DB.GetMetadataCache("composer", cacheKey) + if err != nil || after == nil || after.ContentDigest != stored.ContentDigest { + t.Errorf("cached packages.json replaced by the malformed refresh: %v, %v", after, err) + } + // A fresh handler, as after a restart, has nothing in memory and + // reads the stored copy. + restarted := NewComposerHandlerWithUpstreams(h.proxy, "http://proxy.test", upstream.URL, upstream.URL) + assertComposerSecurityAdvertised(t, restarted, "after a restart") + }) + } +} + +func TestComposerServiceIndexFetchedOnce(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + var wg sync.WaitGroup + for range 10 { + wg.Go(func() { serveComposer(h, "/packages.json") }) + } + wg.Wait() + if got := len(upstream.requestsTo("/packages.json")); got != 1 { + t.Errorf("upstream packages.json requests = %d for parallel requests, want 1", got) + } +} + +func serveAdvisories(h *ComposerHandler, method, target, body string) *httptest.ResponseRecorder { + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req := httptest.NewRequest(method, target, reader) + if method == http.MethodPost { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + return w +} + +func TestComposerAdvisoriesWithholdPrivateNames(t *testing.T) { + // The form Composer sends: PHP's http_build_query numbers the entries. + form := url.Values{ + "packages[0]": {"public/library"}, + "packages[1]": {"example/private"}, + "packages[2]": {"Acme/hosted"}, + "packages[3]": {"other/library"}, + } + tests := []struct { + name, method, target, body string + }{ + {"POST form", http.MethodPost, "/api/security-advisories/", form.Encode()}, + {"GET query", http.MethodGet, "/api/security-advisories/?" + form.Encode(), ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveAdvisories(h, tt.method, tt.target, tt.body) + + if w.Code != http.StatusOK || w.Body.String() != composerAdvisoriesResponse { + t.Fatalf("got %d %q, want upstream's advisories", w.Code, w.Body) + } + sent := upstream.requestsTo("/api/security-advisories/") + if len(sent) != 1 { + t.Fatalf("upstream got %d advisories requests, want 1", len(sent)) + } + if sent[0].method != tt.method { + t.Errorf("upstream method = %s, want %s", sent[0].method, tt.method) + } + query := sent[0].query + if tt.method == http.MethodPost { + query, _ = url.ParseQuery(sent[0].body) + } + got := query["packages[]"] + slices.Sort(got) + if want := []string{"other/library", "public/library"}; !slices.Equal(got, want) { + t.Errorf("upstream was asked about %v, want only the public packages %v", got, want) + } + }) + } +} + +func TestComposerAdvisoriesOnlyPrivateNames(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B0%5D=example%2Fprivate") + + if w.Code != http.StatusOK || w.Body.String() != `{"advisories":{}}` { + t.Errorf("got %d %q, want an empty advisories answer", w.Code, w.Body) + } + if sent := upstream.requestsTo("/api/security-advisories/"); len(sent) != 0 { + t.Errorf("upstream was asked although every package is private: %+v", sent[0]) + } +} + +func TestComposerAdvisoriesUpdatedSince(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveAdvisories(h, http.MethodGet, "/api/security-advisories/?updatedSince=1700000000", "") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + sent := upstream.requestsTo("/api/security-advisories/") + if len(sent) != 1 || sent[0].query.Get("updatedSince") != "1700000000" { + t.Errorf("updatedSince was not passed on: %+v", sent) + } +} + +func TestComposerAdvisoriesNotOffered(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + upstream.index = `{"packages": []}` + h := newAdvisoryComposerHandler(t, upstream) + + if w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B%5D=public%2Flibrary"); w.Code != http.StatusNotFound { + t.Errorf("advisories status = %d, want 404 when upstream has no API", w.Code) + } + if w := serveComposer(h, "/lists/summary.json"); w.Code != http.StatusNotFound { + t.Errorf("summary status = %d, want 404 when upstream has no filter", w.Code) + } +} + +func TestComposerFilterSummaryRelayed(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveComposer(h, "/lists/summary.json") + if w.Code != http.StatusOK || w.Body.String() != composerFilterSummary { + t.Fatalf("got %d %q, want upstream's summary", w.Code, w.Body) + } + + req := httptest.NewRequest(http.MethodGet, "/lists/summary.json", nil) + req.Header.Set("If-Modified-Since", "Wed, 21 Oct 2026 15:45:01 GMT") + w = httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusNotModified { + t.Errorf("revalidation status = %d, want 304 passed on from upstream", w.Code) + } +} + +func TestComposerMetadataKeepsAdvisoriesAndFilter(t *testing.T) { + h := NewComposerHandler(testProxy(), "http://proxy.test") + body := []byte(`{ + "minified": "composer/2.0", + "packages": {"evil/package": [{"name": "evil/package", "version": "1.0.0", "dist": {"url": "https://example.com/evil.zip", "type": "zip"}}]}, + "security-advisories": [{"advisoryId": "PKSA-1", "affectedVersions": "<2.0.0"}], + "filter": {"malware": [{"constraint": "1.0.0", "reason": "malware", "id": "PKFE-1"}]} + }`) + + out, _, err := h.rewriteMetadataKeeping(body) + if err != nil { + t.Fatalf("rewrite: %v", err) + } + var doc map[string]json.RawMessage + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("parsing rewritten metadata: %v", err) + } + if !strings.Contains(string(doc["security-advisories"]), "PKSA-1") { + t.Errorf("security-advisories lost in rewrite: %s", out) + } + if !strings.Contains(string(doc["filter"]), "PKFE-1") { + t.Errorf("filter lost in rewrite: %s", out) + } +} diff --git a/internal/handler/composer_download_lookup_test.go b/internal/handler/composer_download_lookup_test.go index 5635000..53a0cfe 100644 --- a/internal/handler/composer_download_lookup_test.go +++ b/internal/handler/composer_download_lookup_test.go @@ -11,23 +11,25 @@ import ( "time" "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" ) // composerLookupUpstream serves v/p's metadata listing versions and the // archives they point at, counts metadata and archive requests, and fails -// every request while down is set. A version listed in published carries that -// publish time. +// every request while down is set, with downStatus or 503. A version listed in +// published carries that publish time. type composerLookupUpstream struct { - h http.Handler - handler *ComposerHandler - proxy *Proxy - versions atomic.Value // []string - published map[string]time.Time - down atomic.Bool - metadata atomic.Int32 - archives atomic.Int32 - routeURL string + h http.Handler + handler *ComposerHandler + proxy *Proxy + versions atomic.Value // []string + published map[string]time.Time + down atomic.Bool + downStatus atomic.Int32 + metadata atomic.Int32 + archives atomic.Int32 + routeURL string } func setupComposerLookup(t *testing.T, cacheMetadata bool, versions ...string) *composerLookupUpstream { @@ -37,7 +39,12 @@ func setupComposerLookup(t *testing.T, cacheMetadata bool, versions ...string) * var srv *httptest.Server srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if u.down.Load() { - http.Error(w, "down", http.StatusServiceUnavailable) + status := int(u.downStatus.Load()) + if status == 0 { + status = http.StatusServiceUnavailable + } + w.Header().Set("Retry-After", "30") + http.Error(w, "down", status) return } switch { @@ -146,6 +153,42 @@ func TestComposerDownloadWithoutMetadataCache(t *testing.T) { } } +// TestComposerDownloadLookupUpstreamError checks that a download whose +// metadata lookup fails upstream answers 502, or 503 while upstream holds +// requests, and not 404: Composer treats 404 as the version being gone. +func TestComposerDownloadLookupUpstreamError(t *testing.T) { + for _, status := range []int{http.StatusInternalServerError, http.StatusServiceUnavailable} { + t.Run(http.StatusText(status), func(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + u.downStatus.Store(int32(status)) + u.down.Store(true) + if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusBadGateway { + t.Errorf("status = %d, want 502", w.Code) + } + }) + } + + t.Run("rate limited", func(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + gate := registryclient.NewRateLimitGate(http.DefaultTransport) + u.proxy.HTTPClient = &http.Client{Transport: gate} + u.proxy.RateLimits = gate + u.downStatus.Store(http.StatusTooManyRequests) + u.down.Store(true) + w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip") + if w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" { + t.Errorf("status = %d, Retry-After %q; want 503 with Retry-After", w.Code, w.Header().Get("Retry-After")) + } + }) + + t.Run("not listed", func(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + if w := u.get(t, "/files/v/p/2.0.0/2.0.0.zip"); w.Code != http.StatusNotFound { + t.Errorf("status = %d, want 404", w.Code) + } + }) +} + // storedPublishTime returns the publish time recorded for v/p at version. func (u *composerLookupUpstream) storedPublishTime(t *testing.T, version string) (time.Time, bool) { t.Helper() diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index e94d11c..be8a1d6 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -43,6 +43,12 @@ func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([] manifest["name"] = packageName manifest["version"] = lv.Version manifest["time"] = lv.PushedAt.Format(time.RFC3339) + // As for routed packages, keep Composer from reporting installs of + // a private version to the default repository's notify-batch. + // Composer reads a null notification-url as absent too. + if manifest["notification-url"] == nil { + manifest["notification-url"] = "" + } manifest["dist"] = map[string]string{ "type": "zip", "url": h.localDistURL(packageName, lv.Version, lv.Filename), diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index 539e999..dd792ca 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -243,3 +243,28 @@ func TestComposerDownloadOfPushedArchiveWhileStreamingArtifacts(t *testing.T) { t.Error("a pushed archive was fetched upstream while streaming artifacts") } } + +func TestComposerPushedVersionsDisableDefaultNotification(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + f.proxy.HostedComposerVendors = []string{"acme"} + f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) + f.push(t, "acme/private", "1.0.0", `{"name":"acme/private"}`) + f.push(t, "other/private", "1.0.0", `{"name":"other/private"}`) + // Composer reads a null notification-url as absent. + f.push(t, "acme/nulled", "1.0.0", `{"name":"acme/nulled","notification-url":null}`) + f.push(t, "acme/own", "1.0.0", `{"name":"acme/own","notification-url":"https://acme.example.com/notify"}`) + + for _, tc := range []struct{ path, name, version, want string }{ + {"/p2/monolog/monolog.json", "monolog/monolog", "3.6.0", ""}, + {"/p2/acme/private.json", "acme/private", "1.0.0", ""}, + {"/p2/other/private.json", "other/private", "1.0.0", ""}, + {"/p2/acme/nulled.json", "acme/nulled", "1.0.0", ""}, + // A manifest's own URL is kept. + {"/p2/acme/own.json", "acme/own", "1.0.0", "https://acme.example.com/notify"}, + } { + v := f.versions(t, tc.path, tc.name)[tc.version] + if got, ok := v["notification-url"]; !ok || got != tc.want { + t.Errorf("%s %s notification-url = %v (set %v), want %q", tc.name, tc.version, got, ok, tc.want) + } + } +} diff --git a/internal/handler/composer_routes_test.go b/internal/handler/composer_routes_test.go index 5a65440..0ebf5ac 100644 --- a/internal/handler/composer_routes_test.go +++ b/internal/handler/composer_routes_test.go @@ -235,13 +235,16 @@ func TestComposerRoutedDownloadIgnoresDefaultRepositoryArtifact(t *testing.T) { // TestComposerRoutedMinifiedMetadataDisablesNotification checks that every // version of a routed package, as Composer expands minified metadata, has an // empty notification-url: versions that inherit their fields, versions after a -// ~dev reset, and versions after one that set its own URL and then unset it. +// ~dev reset, versions after one that set its own URL and then unset it, and +// versions with a null URL. func TestComposerRoutedMinifiedMetadataDisablesNotification(t *testing.T) { minified := `{"minified":"composer/2.0","packages":{"example/library":[` + `{"name":"example/library","version":"1.0.0","dist":{"url":"https://private.example.com/a/1.0.0.zip","type":"zip"}},` + `{"version":"1.1.0","dist":{"url":"https://private.example.com/a/1.1.0.zip","type":"zip"}},` + `{"version":"1.2.0","notification-url":"https://private.example.com/notify","dist":{"url":"https://private.example.com/a/1.2.0.zip","type":"zip"}},` + `{"version":"1.3.0","notification-url":"__unset","dist":{"url":"https://private.example.com/a/1.3.0.zip","type":"zip"}},` + + `{"version":"1.4.0","notification-url":null,"dist":{"url":"https://private.example.com/a/1.4.0.zip","type":"zip"}},` + + `{"version":"1.5.0","dist":{"url":"https://private.example.com/a/1.5.0.zip","type":"zip"}},` + `"~dev",` + `{"name":"example/library","version":"dev-main","dist":{"url":"https://private.example.com/a/main.zip","type":"zip"}}]}}` public := newComposerUpstream(t, map[string]string{}, http.StatusOK) @@ -258,6 +261,8 @@ func TestComposerRoutedMinifiedMetadataDisablesNotification(t *testing.T) { "1.1.0": "", "1.2.0": "https://private.example.com/notify", // the route's own URL is kept "1.3.0": "", + "1.4.0": "", // Composer reads null as absent + "1.5.0": "", "dev-main": "", } versions := composerVersions(t, w.Body.Bytes(), "example/library") diff --git a/internal/handler/conan.go b/internal/handler/conan.go index dddd79f..90edcdb 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -2,6 +2,7 @@ package handler import ( "net/http" + "slices" "strings" "github.com/git-pkgs/proxy/internal/retention" @@ -79,6 +80,11 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request) revision := r.PathValue("revision") filename := r.PathValue("filename") + if conanPartHasSlash(user, channel, revision, filename) { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + // Only the archives are artifacts; the rest goes through the metadata cache. if !h.shouldCacheFile(filename) { h.proxyCached(w, r) @@ -89,7 +95,13 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request) "name", name, "version", version, "user", user, "channel", channel, "filename", filename) upstreamURL := h.upstreamURL + r.URL.Path - storageFilename := conanStorageFilename(user, channel, revision+"_"+filename) + storageFilename, err := h.proxy.cacheFilenameKeepingLegacy("conan", name, version, + conanStorageFilename(user, channel, revision, filename), + conanLegacyStorageFilename(user, channel, revision+"_"+filename), upstreamURL) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch file") + return + } result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL) if err != nil { @@ -111,6 +123,11 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) pkgrev := r.PathValue("pkgrev") filename := r.PathValue("filename") + if conanPartHasSlash(user, channel, revision, pkgref, pkgrev, filename) { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + // Only the archives are artifacts; the rest goes through the metadata cache. if !h.shouldCacheFile(filename) { h.proxyCached(w, r) @@ -121,7 +138,13 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) "name", name, "version", version, "pkgref", pkgref, "filename", filename) upstreamURL := h.upstreamURL + r.URL.Path - storageFilename := conanStorageFilename(user, channel, revision+"_"+pkgref+"_"+pkgrev+"_"+filename) + storageFilename, err := h.proxy.cacheFilenameKeepingLegacy("conan", name, version, + conanStorageFilename(user, channel, revision, "packages", pkgref, pkgrev, filename), + conanLegacyStorageFilename(user, channel, revision+"_"+pkgref+"_"+pkgrev+"_"+filename), upstreamURL) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch file") + return + } result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL) if err != nil { @@ -133,9 +156,25 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) } // conanStorageFilename names a cached archive. Archives are stored under -// pkg:conan/{name}@{version}, so the file name carries the revisions and, when -// set, user and channel, to keep every recipe and binary apart. -func conanStorageFilename(user, channel, file string) string { +// pkg:conan/{name}@{version}, so the file name carries user, channel and the +// revisions, to keep every recipe and binary apart. The handlers refuse a +// part that contains a slash, so joining them with one keeps different +// references apart. +func conanStorageFilename(user, channel string, parts ...string) string { + return strings.Join(append([]string{user, channel}, parts...), "/") +} + +// conanPartHasSlash reports whether a path value contains a slash, which it +// can only get from an encoded one (%2F) in the request. +func conanPartHasSlash(parts ...string) bool { + return slices.ContainsFunc(parts, func(part string) bool { return strings.Contains(part, "/") }) +} + +// conanLegacyStorageFilename is the name archives were cached under before +// their parts were joined with slashes. Underscores, which revisions, user +// and channel can contain, joined them, so it is only read for the URL an +// entry was fetched from. +func conanLegacyStorageFilename(user, channel, file string) string { if user == "_" && channel == "_" { return file } diff --git a/internal/handler/conan_test.go b/internal/handler/conan_test.go index cf8038e..200f867 100644 --- a/internal/handler/conan_test.go +++ b/internal/handler/conan_test.go @@ -1,6 +1,7 @@ package handler import ( + "context" "io" "log/slog" "net/http" @@ -9,6 +10,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/git-pkgs/registries/fetch" ) const testProxyURL = "http://localhost:8080" @@ -302,13 +305,13 @@ func TestConanV2FileRoutesCacheArchives(t *testing.T) { filename string }{ {"recipe", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", - "rrev1_conan_sources.tgz"}, + "_/_/rrev1/conan_sources.tgz"}, {"package", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tgz", - "rrev1_pkgid1_prev1_conan_package.tgz"}, + "_/_/rrev1/packages/pkgid1/prev1/conan_package.tgz"}, {"recipe xz", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_export.txz", - "rrev1_conan_export.txz"}, + "_/_/rrev1/conan_export.txz"}, {"package zstd", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tzst", - "rrev1_pkgid1_prev1_conan_package.tzst"}, + "_/_/rrev1/packages/pkgid1/prev1/conan_package.tzst"}, } for _, tt := range tests { @@ -362,9 +365,9 @@ func TestConanUserChannelKeptApart(t *testing.T) { for _, ref := range []struct{ path, body, filename string }{ {"/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", - "center sources", "rrev1_conan_sources.tgz"}, + "center sources", "_/_/rrev1/conan_sources.tgz"}, {"/v2/conans/zlib/1.3.1/acme/stable/revisions/rrev1/files/conan_sources.tgz", - "acme sources", "acme@stable_rrev1_conan_sources.tgz"}, + "acme sources", "acme/stable/rrev1/conan_sources.tgz"}, } { fetcher.artifact = artifactBody(ref.body) fetcher.fetchCalled = false @@ -651,3 +654,102 @@ func TestConanMetadataPreservesStatusCodes(t *testing.T) { upstream.Close() } } + +const ( + conanRecipeLookalike = "/v2/conans/zlib/1.3/_/_/revisions/R_P_Q/files/conan_package.tgz" + conanPackageFile = "/v2/conans/zlib/1.3/_/_/revisions/R/packages/P/revisions/Q/files/conan_package.tgz" +) + +// conanPathEchoServer answers every file with its own path, so each URL has +// different bytes. +func conanPathEchoServer(t *testing.T, available *atomic.Bool) *httptest.Server { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte(r.URL.Path)) + })) + t.Cleanup(upstream.Close) + return upstream +} + +func serveConanRequest(h *ConanHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} + +// A recipe revision with underscores must not share a cache entry with a +// package whose revision, reference and package revision spell it. +func TestConanFileKeysKeepPathFieldsApart(t *testing.T) { + var available atomic.Bool + available.Store(true) + upstream := conanPathEchoServer(t, &available) + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) + + for _, target := range []string{conanRecipeLookalike, conanPackageFile} { + if w := serveConanRequest(h, target); w.Code != http.StatusOK || w.Body.String() != target { + t.Errorf("GET %s = %d %q, want its own file", target, w.Code, w.Body.String()) + } + } +} + +// A file cached under the name used before path fields were kept apart is +// still served for the URL it was fetched from, and only for that URL. +func TestConanLegacyFileEntryServedForItsURL(t *testing.T) { + var available atomic.Bool + available.Store(true) + upstream := conanPathEchoServer(t, &available) + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) + + legacy, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "conan", "zlib", "1.3", + "R_P_Q_conan_package.tgz", upstream.URL+conanRecipeLookalike) + if err != nil { + t.Fatalf("seed legacy entry: %v", err) + } + _ = legacy.Reader.Close() + + if w := serveConanRequest(h, conanPackageFile); w.Body.String() != conanPackageFile { + t.Errorf("package file = %d %q, want its own file, not the legacy entry", w.Code, w.Body.String()) + } + available.Store(false) + if w := serveConanRequest(h, conanRecipeLookalike); w.Code != http.StatusOK || w.Body.String() != conanRecipeLookalike { + t.Errorf("recipe file with the upstream down = %d %q, want the legacy entry", w.Code, w.Body.String()) + } +} + +// A path value is decoded, so one holding an encoded slash could spell the +// cache key of another reference. Those requests are refused. +func TestConanFileRoutesRefuseEncodedSlash(t *testing.T) { + var available atomic.Bool + available.Store(true) + upstream := conanPathEchoServer(t, &available) + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) + + if w := serveConanRequest(h, conanPackageFile); w.Code != http.StatusOK || w.Body.String() != conanPackageFile { + t.Fatalf("GET %s = %d %q, want its own file", conanPackageFile, w.Code, w.Body.String()) + } + + const recipe, pkg = "/v2/conans/zlib/1.3/_/_/revisions/", "/v2/conans/zlib/1.3/_/_/revisions/R/packages/" + for field, target := range map[string]string{ + "revision spelling a package file": recipe + "R%2Fpackages%2FP%2FQ/files/conan_package.tgz", + "user": "/v2/conans/zlib/1.3/a%2Fb/_/revisions/R/files/conan_package.tgz", + "channel": "/v2/conans/zlib/1.3/_/a%2Fb/revisions/R/files/conan_package.tgz", + "recipe filename": recipe + "R/files/a%2Fconan_package.tgz", + "package reference": pkg + "P%2FX/revisions/Q/files/conan_package.tgz", + "package revision": pkg + "P/revisions/Q%2FX/files/conan_package.tgz", + "package filename": pkg + "P/revisions/Q/files/a%2Fconan_package.tgz", + } { + if w := serveConanRequest(h, target); w.Code != http.StatusBadRequest { + t.Errorf("%s: GET %s = %d %q, want 400", field, target, w.Code, w.Body.String()) + } + } +} diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 3cbaf93..c9db7a7 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -139,11 +139,21 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("conda download request", "channel", channel, "arch", arch, "name", name, "version", version) - upstreamURL := h.upstreamURL + r.URL.Path - // The same filename can hold different bytes in each subdir, so the - // subdir is part of the cache key. + // Builds and subdirs of one version each have their own timestamp, so + // the check is per file and keeps no time in the shared versions row. + // Subdirs distinguish cached files with the same filename. cacheFilename := arch + "/" + filename + if h.proxy.withholdInCooldown(w, r, downloadVersion{ + ecosystem: "conda", policyName: name, version: version, cacheName: packageName, filename: cacheFilename, + lookup: func() (time.Time, bool, bool, error) { + published, ok, err := h.publishTime(r, channel, arch, filename) + return published, ok, false, err + }, + }) { + return + } + upstreamURL := h.upstreamURL + r.URL.Path result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, cacheFilename, upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") @@ -409,7 +419,8 @@ func (h *CondaHandler) rememberRepodata(cacheKey, digest string, filter *repodat h.repodataStates[cacheKey] = repodataState{digest: digest, etag: etag, until: filter.until} } -// countingWriter records whether anything has reached the response yet. +// countingWriter counts the bytes written through it, such as whether +// anything has reached the response yet. type countingWriter struct { w io.Writer n int64 @@ -421,8 +432,18 @@ func (c *countingWriter) Write(p []byte) (int, error) { return n, err } -// condaTimestampDivisor converts Conda's millisecond timestamps to seconds. -const condaTimestampDivisor = 1000 +// condaMaxSecondsTimestamp is the last second of year 9999. Like conda, a +// larger repodata timestamp is read as milliseconds and a smaller one as +// seconds, which some older entries use. +const condaMaxSecondsTimestamp = 253402300799 + +// condaTime returns the time a repodata timestamp names. +func condaTime(timestamp float64) time.Time { + if timestamp > condaMaxSecondsTimestamp { + return time.UnixMilli(int64(timestamp)).Truncate(time.Second) + } + return time.Unix(int64(timestamp), 0) +} const ( repodataWriteBuffer = 64 << 10 @@ -440,7 +461,7 @@ func (h *CondaHandler) filterRepodata(r io.Reader, w io.Writer) error { // filterRepodataWith is filterRepodata deciding by filter. func (h *CondaHandler) filterRepodataWith(r io.Reader, w io.Writer, filter *repodataFilter) error { - return jsonstream.CopyObject(json.NewDecoder(r), w, func(m *jsonstream.Member) error { + return jsonstream.CopyObject(r, w, func(m *jsonstream.Member) error { if m.Key != "packages" && m.Key != "packages.conda" { return m.Copy() } @@ -492,7 +513,7 @@ func (f *repodataFilter) inCooldown(filename string, raw []byte) bool { if entry.Name == "" || entry.Timestamp == 0 { return false } - publishedAt := time.Unix(int64(entry.Timestamp)/condaTimestampDivisor, 0) + publishedAt := condaTime(entry.Timestamp) decision := f.h.proxy.cooldown().Evaluate("conda", canonicalPackagePURL("conda", entry.Name), publishedAt, f.now) if decision.Allowed { return false @@ -509,14 +530,14 @@ func (f *repodataFilter) inCooldown(filename string, raw []byte) bool { return true } -// recordInCooldown reports whether a package record with a millisecond +// recordInCooldown reports whether a package record with a repodata // timestamp was published too recently. Records without a name or // timestamp are kept. func (h *CondaHandler) recordInCooldown(filename, name, version string, timestamp float64) bool { if name == "" || timestamp == 0 { return false } - publishedAt := time.Unix(int64(timestamp)/condaTimestampDivisor, 0) + publishedAt := condaTime(timestamp) if h.proxy.cooldown().IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) { return false } @@ -530,6 +551,96 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { return hashedCacheKey(h.upstreamURL, urlPath) } +// publishTime streams compressed repodata when it exceeds metadata_max_size. +func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { + repodataPath := "/" + channel + "/" + arch + "/repodata.json" + cacheKey := h.metadataCacheKey(repodataPath) + index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, func() metadataResult { + return h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) + }, func(res metadataResult) (map[string]time.Time, error) { + if errors.Is(res.err, ErrMetadataTooLarge) { + return h.streamRepodataIndex(r, h.upstreamURL+repodataPath) + } + if res.err != nil { + return nil, res.err + } + return h.indexRepodata(res.body, res.contentEncoding) + }) + if err != nil { + return time.Time{}, false, err + } + published, ok := index[filename] + return published, ok, nil +} + +// indexRepodata indexes a cached repodata.json body. +func (h *CondaHandler) indexRepodata(body []byte, encoding string) (map[string]time.Time, error) { + var reader io.Reader = bytes.NewReader(body) + if strings.EqualFold(strings.TrimSpace(encoding), "gzip") { + decoded, err := gzip.NewReader(reader) + if err != nil { + return nil, err + } + reader = decoded + } + return readRepodataIndex(reader) +} + +// streamRepodataIndex indexes repodata.json as it streams from upstream. +func (h *CondaHandler) streamRepodataIndex(r *http.Request, upstreamURL string) (map[string]time.Time, error) { + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) + if err != nil { + return nil, err + } + req.Header.Set(headerAcceptEncoding, "gzip") + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("repodata returned %d", resp.StatusCode) + } + var reader io.Reader = resp.Body + if strings.EqualFold(strings.TrimSpace(resp.Header.Get(headerContentEncoding)), "gzip") { + decoded, err := gzip.NewReader(resp.Body) + if err != nil { + return nil, err + } + defer func() { _ = decoded.Close() }() + reader = decoded + } + return readRepodataIndex(reader) +} + +// readRepodataIndex maps each package file in a repodata.json to its +// timestamp, leaving out files without one. It reads the document as it +// streams, under the same bound as filterRepodata, so the decompressed size +// is not limited by metadata_max_size. +func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { + index := make(map[string]time.Time) + err := jsonstream.CopyObject(io.LimitReader(r, maxRepodataSize), io.Discard, func(m *jsonstream.Member) error { + if m.Key != "packages" && m.Key != "packages.conda" { + return m.Copy() + } + return m.Object(func(entry *jsonstream.Member) error { + raw, err := entry.Raw() + if err != nil { + return err + } + var e repodataEntry + if json.Unmarshal(raw, &e) == nil && e.Timestamp != 0 { + index[entry.Key] = condaTime(e.Timestamp) + } + return nil + }) + }) + if err != nil { + return nil, err + } + return index, nil +} + // proxyCached forwards a metadata request with caching. func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) { cacheKey := h.metadataCacheKey(r.URL.Path) @@ -540,7 +651,7 @@ func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, ".json") { acceptEncoding = "gzip" } - h.proxy.proxyCachedWithEncoding(w, r, h.upstreamURL+r.URL.Path, "conda", cacheKey, acceptEncoding, "*/*") + h.proxy.proxyCachedWithEncoding(w, r, h.upstreamURL+r.URL.Path, "conda", cacheKey, acceptEncoding, nil, "*/*") } // proxyUpstream forwards a request to Anaconda without caching. diff --git a/internal/handler/conda_repodata_bench_test.go b/internal/handler/conda_repodata_bench_test.go new file mode 100644 index 0000000..61e68bd --- /dev/null +++ b/internal/handler/conda_repodata_bench_test.go @@ -0,0 +1,59 @@ +package handler + +import ( + "bytes" + "io" + "log/slog" + "os" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// repodataBenchBody reads the repodata.json named by CONDA_REPODATA_BENCH, +// such as conda-forge's noarch one, and skips the benchmark without it. +func repodataBenchBody(b *testing.B) []byte { + b.Helper() + path := os.Getenv("CONDA_REPODATA_BENCH") + if path == "" { + b.Skip("CONDA_REPODATA_BENCH names no repodata.json") + } + body, err := os.ReadFile(path) + if err != nil { + b.Fatal(err) + } + return body +} + +func BenchmarkCondaFilterRepodata(b *testing.B) { + body := repodataBenchBody(b) + policy, err := cooldown.NewPolicy(&cooldown.Config{Default: "3d"}, nil) + if err != nil { + b.Fatal(err) + } + proxy := testProxy() + proxy.Logger = slog.New(slog.DiscardHandler) + proxy.Cooldown = policy + h := NewCondaHandler(proxy, "http://proxy.example") + now := time.Now() + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if err := h.filterRepodataWith(bytes.NewReader(body), io.Discard, h.newRepodataFilter(now)); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCondaReadRepodataIndex(b *testing.B) { + body := repodataBenchBody(b) + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + index, err := readRepodataIndex(bytes.NewReader(body)) + if err != nil || len(index) == 0 { + b.Fatalf("index of %d files, %v", len(index), err) + } + } +} diff --git a/internal/handler/conda_shards.go b/internal/handler/conda_shards.go index 01434e4..dbfaee2 100644 --- a/internal/handler/conda_shards.go +++ b/internal/handler/conda_shards.go @@ -58,11 +58,12 @@ func (h *CondaHandler) handleShardIndex(w http.ResponseWriter, r *http.Request) upstreamURL := h.upstreamURL + r.URL.Path cacheKey := h.metadataCacheKey(r.URL.Path) - body, contentType, contentEncoding, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", cacheKey, upstreamURL, "identity", nil, "*/*") - if err != nil { - h.proxy.writeMetadataFetchError(w, err) + res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "identity", nil) + if res.err != nil { + h.proxy.writeMetadataFetchError(w, res.err) return } + body := res.body if !h.cooldownEnabled() { index, err := decodeCondaShardIndex(body) @@ -74,7 +75,7 @@ func (h *CondaHandler) handleShardIndex(w http.ResponseWriter, r *http.Request) http.NotFound(w, r) return } - h.proxy.writeMetadataCachedResponseWithEncoding(w, r, "conda", cacheKey, body, contentType, contentEncoding) + writeMetadataResult(w, r, res) return } @@ -121,7 +122,7 @@ func (h *CondaHandler) handleCooldownShard(w http.ResponseWriter, r *http.Reques } upstreamPath := "/" + channel + "/" + arch + "/" + dir + hex.EncodeToString(masked) + condaShardSuffix - body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", h.metadataCacheKey(upstreamPath), h.upstreamURL+upstreamPath, "identity", nil, "*/*") + body, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", h.metadataCacheKey(upstreamPath), h.upstreamURL+upstreamPath, "identity", nil, "*/*") if err != nil { h.proxy.writeMetadataFetchError(w, err) return @@ -378,7 +379,8 @@ func rewriteCondaShardIndex(body []byte, indexURL string, mask [condaShardHashLe } // condaShardRecord holds the fields cooldown needs from a shard record. -// Timestamps are milliseconds, encoded as an integer or a float. +// Timestamps are an integer or a float, in seconds or milliseconds as +// condaTime reads them. type condaShardRecord struct { Name string `msgpack:"name"` Version string `msgpack:"version"` diff --git a/internal/handler/conda_test.go b/internal/handler/conda_test.go index 0c4c52a..f6c37c6 100644 --- a/internal/handler/conda_test.go +++ b/internal/handler/conda_test.go @@ -3,9 +3,13 @@ package handler import ( "bytes" "encoding/json" + "fmt" "log/slog" + "maps" "net/http" "net/http/httptest" + "slices" + "strings" "testing" "time" @@ -133,6 +137,45 @@ func TestCondaCooldownFiltering(t *testing.T) { } } +// TestCondaCooldownSecondTimestamps checks that a timestamp in seconds, +// which some repodata entries carry, is read as seconds, as conda does, in +// both the filter and the download index. +func TestCondaCooldownSecondTimestamps(t *testing.T) { + recent := time.Now().Add(-time.Hour) + body := fmt.Sprintf(`{"packages":{"seconds-1.0-0.tar.bz2":{"name":"seconds","version":"1.0","timestamp":%d},`+ + `"millis-1.0-0.tar.bz2":{"name":"millis","version":"1.0","timestamp":%d}}}`, recent.Unix(), recent.UnixMilli()) + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := &CondaHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + filtered, err := filterRepodataBytes(h, []byte(body)) + if err != nil { + t.Fatal(err) + } + var result struct { + Packages map[string]json.RawMessage `json:"packages"` + } + if err := json.Unmarshal(filtered, &result); err != nil { + t.Fatal(err) + } + if len(result.Packages) != 0 { + t.Errorf("packages = %v, want both left out", slices.Sorted(maps.Keys(result.Packages))) + } + + index, err := readRepodataIndex(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if len(index) != 2 { + t.Fatalf("index has %d entries, want 2", len(index)) + } + for filename, published := range index { + if published.Unix() != recent.Unix() { + t.Errorf("%s published at %v, want %v", filename, published, recent) + } + } +} + func TestCondaCooldownFilteringWithPackageOverride(t *testing.T) { now := time.Now() recentTimestamp := float64(now.Add(-2 * time.Hour).UnixMilli()) diff --git a/internal/handler/container.go b/internal/handler/container.go index d9360fb..40f5d0f 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -353,7 +353,7 @@ func (h *ContainerHandler) Routes() http.Handler { case path == "" || path == "/": // Version check: GET /v2/ h.handleVersionCheck(w, r) - case strings.HasSuffix(path, "/blobs/"+r.URL.Query().Get("digest")) || strings.Contains(path, "/blobs/sha256:"): + case strings.HasSuffix(path, "/blobs/"+r.URL.Query().Get("digest")) || strings.Contains(path, "/blobs/sha256:") || strings.Contains(path, "/blobs/sha512:"): // Blob download: GET /v2/{name}/blobs/{digest} h.handleBlobDownload(w, r, path) case manifestPathPattern.MatchString(path): @@ -692,7 +692,7 @@ func (h *ContainerHandler) containerError(w http.ResponseWriter, status int, cod } // blobPathPattern matches blob paths: {name}/blobs/{digest} -var blobPathPattern = regexp.MustCompile(`^(.+)/blobs/(sha256:[a-f0-9]+)$`) +var blobPathPattern = regexp.MustCompile(`^(.+)/blobs/(sha256:[a-f0-9]{64}|sha512:[a-f0-9]{128})$`) // parseBlobPath extracts repository name and digest from a blob path. func (h *ContainerHandler) parseBlobPath(path string) (name, digest string) { diff --git a/internal/handler/container_manifest_fetch.go b/internal/handler/container_manifest_fetch.go index 4b5ef15..7568671 100644 --- a/internal/handler/container_manifest_fetch.go +++ b/internal/handler/container_manifest_fetch.go @@ -210,6 +210,11 @@ func (h *ContainerHandler) headManifest(ctx context.Context, req manifestRequest digest := resp.Header.Get("Docker-Content-Digest") switch { case resp.StatusCode == http.StatusOK && manifestDigestReferencePattern.MatchString(digest): + if req.immutable() && digestMismatch(req.reference, digest) { + h.proxy.Logger.Error("upstream manifest HEAD names another digest", + "name", req.name, "reference", req.reference, "upstream", digest) + return manifestOutcome{digestInvalid: true} + } if !req.immutable() { if outcome, ok := h.tagFromCachedDigest(ctx, req, cached, fromFallback, resp, digest); ok { return outcome @@ -234,6 +239,14 @@ func (h *ContainerHandler) headManifest(ctx context.Context, req manifestRequest return manifestOutcome{relay: h.captureResponse(resp)} } +// digestMismatch reports whether two digests of the same algorithm differ. +// Digests of different algorithms can't be compared without the body. +func digestMismatch(requested, upstream string) bool { + reqAlg, reqHex, _ := strings.Cut(requested, ":") + upAlg, upHex, _ := strings.Cut(upstream, ":") + return reqAlg == upAlg && !strings.EqualFold(reqHex, upHex) +} + // tagFromCachedDigest answers a tag from the cache once a HEAD named the // digest it points to: the cached tag entry if the tag did not move, or the // manifest stored under the new digest. It reports false when neither is diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index ddd3022..f19ac73 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -11,7 +11,11 @@ import ( "regexp" "slices" "strings" + "sync" + "sync/atomic" "time" + + "golang.org/x/sync/errgroup" ) const ( @@ -19,6 +23,9 @@ const ( maxMirrorIndexDepth = 2 // mirrorErrorBodyLimit bounds how much of an error response is quoted. mirrorErrorBodyLimit = 200 + // mirrorBlobConcurrency is how many blobs of one manifest are fetched at + // once, as many as a Docker pull fetches by default. + mirrorBlobConcurrency = 3 dockerReferenceTypeAnnotation = "vnd.docker.reference.type" dockerReferenceDigestAnnotation = "vnd.docker.reference.digest" @@ -29,7 +36,7 @@ var ( // Grammar from the OCI distribution spec. mirrorNamePattern = regexp.MustCompile(`^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*$`) mirrorTagPattern = regexp.MustCompile(`^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$`) - mirrorDigestPattern = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`) + mirrorDigestPattern = regexp.MustCompile(`^(sha256:[a-f0-9]{64}|sha512:[a-f0-9]{128})$`) ) // dockerHubMirrorAliases are registry hosts that name Docker Hub in image @@ -325,7 +332,7 @@ func (m *imageMirror) requestTag(ctx context.Context, method, tag, accept string // mirrorManifest stores the manifest for reference through the handler's own // manifest route, then follows what it references. It returns the manifest -// digest. +// sha256 digest. func (m *imageMirror) mirrorManifest(ctx context.Context, reference string, depth int) (string, error) { cacheAccept := normalizeContainerManifestAccept(containerManifestAccept(&http.Request{Header: http.Header{}})) before, _ := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, reference, cacheAccept, cacheAccept) @@ -353,6 +360,9 @@ func (m *imageMirror) mirrorManifest(ctx context.Context, reference string, dept if before == nil || before.contentDigest != digest { m.result.Fetched++ } + // Tags resolve to sha256 digests, so a manifest asked for by another + // digest is tracked by its sha256 too. + digest = sha256Digest(response.body.Bytes()) if m.seen[digest] { return digest, nil } @@ -439,6 +449,7 @@ func (m *imageMirror) mirrorBlobs(ctx context.Context, manifest *ociManifest) er if manifest.Config != nil { blobs = append([]ociDescriptor{*manifest.Config}, blobs...) } + var digests []string for _, blob := range blobs { if len(blob.URLs) > 0 || strings.Contains(blob.MediaType, "nondistributable") || strings.Contains(blob.MediaType, "foreign") { m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("skipped non-distributable layer %s", blob.Digest)) @@ -451,34 +462,65 @@ func (m *imageMirror) mirrorBlobs(ctx context.Context, manifest *ociManifest) er continue } m.seen[blob.Digest] = true - if err := m.mirrorBlob(ctx, blob.Digest); err != nil { - return err - } + digests = append(digests, blob.Digest) } - return nil + + // Large layers take most of the time, so they are fetched side by side. + // A failure starts no further blobs but lets those in flight finish + // rather than cancelling them: a client pull may share their fetch, and + // they would have to be fetched again next run. + var mu sync.Mutex + var failed atomic.Bool + var g errgroup.Group + g.SetLimit(mirrorBlobConcurrency) + for _, digest := range digests { + g.Go(func() error { + if failed.Load() { + return nil + } + size, fetched, err := m.mirrorBlob(ctx, digest) + if err != nil { + failed.Store(true) + return err + } + if !fetched { + return nil + } + mu.Lock() + m.result.Fetched++ + m.result.Bytes += size + mu.Unlock() + return nil + }) + } + return g.Wait() } -// mirrorBlob stores a blob under the same identity handleBlobDownload uses. -func (m *imageMirror) mirrorBlob(ctx context.Context, digest string) error { +// mirrorBlob stores a blob under the same identity handleBlobDownload uses, +// and returns its size when it had to be fetched. +func (m *imageMirror) mirrorBlob(ctx context.Context, digest string) (int64, bool, error) { proxy := m.h.proxy cached, err := proxy.HasCachedArtifact(ctx, "oci", m.cacheName, digest, digest) if err != nil { - return fmt.Errorf("blob %s: %w", digest, err) + return 0, false, fmt.Errorf("blob %s: %w", digest, err) } if cached { - return nil + // Not a cache hit, so LRU eviction still sees the image as unused, + // but retention keeps it while the schedule does. + if err := proxy.MarkMirrored("oci", m.cacheName, digest, digest); err != nil { + return 0, false, fmt.Errorf("blob %s: %w", digest, err) + } + return 0, false, nil } result, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "oci", m.cacheName, digest, digest, fmt.Sprintf("%s/v2/%s/blobs/%s", m.registryURL, m.upstreamName, digest), digest) if err != nil { - return fmt.Errorf("blob %s: %w", digest, err) + return 0, false, fmt.Errorf("blob %s: %w", digest, err) } if result.Reader != nil { _ = result.Reader.Close() } - m.result.Fetched++ - m.result.Bytes += result.Artifact.Size - return nil + return result.Artifact.Size, true, nil } // mirrorResponse collects the handler's manifest response in memory. diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index 149fb23..34af48c 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -2,16 +2,21 @@ package handler import ( "context" + "crypto/sha512" "database/sql" + "encoding/hex" "encoding/json" "io" "net/http" "net/http/httptest" "net/url" + "slices" "strconv" "strings" + "sync" "sync/atomic" "testing" + "time" "github.com/git-pkgs/registries/fetch" ) @@ -19,8 +24,10 @@ import ( // fakeImageRegistry serves manifests and blobs of a small multi-arch image, // a single-arch image and a moved tag. It fails with 503 once offline. type fakeImageRegistry struct { - server *httptest.Server - offline atomic.Bool + server *httptest.Server + offline atomic.Bool + // mu guards manifests against a test changing it while blobs are served. + mu sync.RWMutex manifests map[string]fakeManifest blobs map[string]string blobGets atomic.Int32 @@ -30,6 +37,9 @@ type fakeImageRegistry struct { truncateManifests atomic.Bool // onBlob, if set, runs before a blob is served. onBlob func(path string) + // sha512Digests makes manifests requested by a sha512 digest name that + // digest in Docker-Content-Digest, as registries do. + sha512Digests bool } type fakeImage struct { @@ -117,9 +127,15 @@ func newFakeImageRegistry(t *testing.T) (*fakeImageRegistry, fakeImage) { http.Error(w, "offline", http.StatusServiceUnavailable) return } - if doc, ok := r.manifests[req.URL.Path]; ok { + r.mu.RLock() + doc, ok := r.manifests[req.URL.Path] + r.mu.RUnlock() + if ok { w.Header().Set("Content-Type", doc.contentType) w.Header().Set("Docker-Content-Digest", sha256Digest([]byte(doc.body))) + if _, ref, _ := strings.Cut(req.URL.Path, "/manifests/"); r.sha512Digests && strings.HasPrefix(ref, "sha512:") { + w.Header().Set("Docker-Content-Digest", ref) + } if req.Method == http.MethodGet && !strings.Contains(req.URL.Path, "/manifests/sha256:") { r.tagGets.Add(1) } @@ -432,7 +448,9 @@ func TestMirrorImageKeepsTagMovedByPullDuringWalk(t *testing.T) { var once atomic.Bool registry.onBlob = func(string) { if once.CompareAndSwap(false, true) { + registry.mu.Lock() registry.manifests["/v2/team/app/manifests/single"] = moved + registry.mu.Unlock() if response := pull(t, h, "/team/app/manifests/single", dockerManifestAccept); response.Code != http.StatusOK { t.Errorf("client pull during the walk: status %d", response.Code) } @@ -479,6 +497,43 @@ func TestMirrorImageCountsNoCacheHits(t *testing.T) { } } +// A scheduled image that no client pulls keeps its blobs: a mirror run that +// finds them cached holds them back from retention, though not from LRU +// eviction, which TestMirrorImageCountsNoCacheHits covers. +func TestMirrorImageKeepsCachedBlobsFromRetention(t *testing.T) { + registry, _ := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + ref := ImageRef{Name: "team/app", References: []string{"multi"}} + + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("first MirrorImage: %v", err) + } + old := time.Now().Add(-30 * 24 * time.Hour).UTC() + if _, err := proxy.DB.Exec(proxy.DB.Rebind("UPDATE artifacts SET fetched_at = ?, last_accessed_at = ?"), old, old); err != nil { + t.Fatal(err) + } + cutoff := time.Now().Add(-14 * 24 * time.Hour) + maxID, err := proxy.DB.MaxArtifactID() + if err != nil { + t.Fatal(err) + } + if candidates, err := proxy.DB.GetRetentionCandidates(0, maxID, cutoff); err != nil || len(candidates) == 0 { + t.Fatalf("before the second run: %d candidates, err %v; want the aged blobs", len(candidates), err) + } + + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("second MirrorImage: %v", err) + } + candidates, err := proxy.DB.GetRetentionCandidates(0, maxID, cutoff) + if err != nil { + t.Fatal(err) + } + if len(candidates) != 0 { + t.Errorf("after the second run %d blobs are retention candidates, want none", len(candidates)) + } +} + func TestMirrorImageRefetchesBlobMissingFromStorage(t *testing.T) { registry, image := newFakeImageRegistry(t) proxy := newMirrorTestProxy(t, registry) @@ -503,3 +558,128 @@ func TestMirrorImageRefetchesBlobMissingFromStorage(t *testing.T) { t.Errorf("second run fetched %d blobs, want the one missing from storage", got) } } + +func TestMirrorImageFetchesBlobsConcurrently(t *testing.T) { + registry, _ := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + + // "single" has a config and a layer blob. Each request waits until the + // other is in flight too, or gives up after a while. + var inFlight atomic.Int32 + var concurrent atomic.Bool + both := make(chan struct{}) + var once sync.Once + registry.onBlob = func(string) { + defer inFlight.Add(-1) + if inFlight.Add(1) >= 2 { + concurrent.Store(true) + once.Do(func() { close(both) }) + } + select { + case <-both: + case <-time.After(2 * time.Second): + } + } + result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"single"}}, nil) + if err != nil { + t.Fatalf("MirrorImage: %v", err) + } + if !concurrent.Load() { + t.Error("blobs were fetched one after another, want them fetched concurrently") + } + if result.Fetched != 3 { + t.Errorf("fetched = %d, want the manifest and its two blobs", result.Fetched) + } +} + +// TestMirrorImageBlobFailureKeepsOtherFetches checks that a blob failing +// doesn't cancel the blobs already being fetched: their fetch may be shared +// with a client pull, which would fail with it, and the next run would have +// to fetch them again. +func TestMirrorImageBlobFailureKeepsOtherFetches(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + + // The config blob is gone upstream; the layer is still being sent when + // its 404 arrives. + for path := range registry.blobs { + if !strings.HasSuffix(path, image.singleLayer) { + delete(registry.blobs, path) + } + } + registry.onBlob = func(string) { time.Sleep(300 * time.Millisecond) } + + if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"single"}}, nil); err == nil { + t.Fatal("MirrorImage succeeded with a blob missing upstream") + } + registry.offline.Store(true) + if response := pull(t, h, "/team/app/blobs/"+image.singleLayer, ""); response.Code != http.StatusOK { + t.Errorf("layer fetched alongside the failing blob: status %d, want it cached", response.Code) + } +} + +func TestMirrorImageAcceptsSHA512Blobs(t *testing.T) { + registry, image := newFakeImageRegistry(t) + const layer = "sha512 layer" + sum := sha512.Sum512([]byte(layer)) + layerDigest := "sha512:" + hex.EncodeToString(sum[:]) + body, err := json.Marshal(map[string]any{ + "schemaVersion": 2, "mediaType": ociManifestMediaType, + "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": image.singleLayer}, + "layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": layerDigest}}, + }) + if err != nil { + t.Fatal(err) + } + registry.mu.Lock() + registry.manifests["/v2/team/app/manifests/sha512"] = fakeManifest{contentType: ociManifestMediaType, body: string(body)} + registry.blobs["/v2/team/app/blobs/"+layerDigest] = layer + registry.mu.Unlock() + + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"sha512"}}, nil); err != nil { + t.Fatalf("MirrorImage: %v", err) + } + cached, err := proxy.GetCachedArtifact(t.Context(), "oci", "team/app", layerDigest, layerDigest) + if err != nil || cached == nil { + t.Fatalf("sha512 layer not cached: %v, %v", cached, err) + } + if cached.Reader != nil { + _ = cached.Reader.Close() + } +} + +// TestMirrorImageSHA512ReferenceMatchesTag checks that mirroring a tag +// pinned by a sha512 digest compares it with the tag's manifest by sha512, +// so an unmoved tag is neither reported as moved nor counted twice. +func TestMirrorImageSHA512ReferenceMatchesTag(t *testing.T) { + mirror := func(t *testing.T, sha512Ref bool) ImageResult { + registry, image := newFakeImageRegistry(t) + reference := image.singleDigest + if sha512Ref { + body := registry.manifests["/v2/team/app/manifests/single"].body + sum := sha512.Sum512([]byte(body)) + reference = "sha512:" + hex.EncodeToString(sum[:]) + registry.manifests["/v2/team/app/manifests/"+reference] = registry.manifests["/v2/team/app/manifests/single"] + registry.sha512Digests = true + } + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{reference, "single"}}, nil) + if err != nil { + t.Fatalf("MirrorImage: %v", err) + } + return result + } + want := mirror(t, false) + got := mirror(t, true) + if !slices.Equal(got.Warnings, want.Warnings) { + t.Errorf("warnings = %v, want %v as for a sha256 reference", got.Warnings, want.Warnings) + } + if got.Fetched != want.Fetched { + t.Errorf("fetched = %d, want %d as for a sha256 reference", got.Fetched, want.Fetched) + } +} diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index d546725..9ba1535 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -18,6 +18,11 @@ import ( "github.com/git-pkgs/registries/fetch" ) +var ( + testSHA256Digest = "sha256:" + strings.Repeat("ab", 32) + testSHA512Digest = "sha512:" + strings.Repeat("cd", 64) +) + func TestContainerHandler_parseBlobPath(t *testing.T) { h := &ContainerHandler{} @@ -27,19 +32,29 @@ func TestContainerHandler_parseBlobPath(t *testing.T) { wantDigest string }{ { - path: "library/nginx/blobs/sha256:abc123def456", + path: "library/nginx/blobs/" + testSHA256Digest, wantName: "library/nginx", - wantDigest: "sha256:abc123def456", + wantDigest: testSHA256Digest, }, { - path: "myorg/myrepo/blobs/sha256:0123456789abcdef", - wantName: "myorg/myrepo", - wantDigest: "sha256:0123456789abcdef", - }, - { - path: "deep/nested/repo/name/blobs/sha256:fedcba9876543210", + path: "deep/nested/repo/name/blobs/" + testSHA256Digest, wantName: "deep/nested/repo/name", - wantDigest: "sha256:fedcba9876543210", + wantDigest: testSHA256Digest, + }, + { + path: "myorg/myrepo/blobs/" + testSHA512Digest, + wantName: "myorg/myrepo", + wantDigest: testSHA512Digest, + }, + { + path: "library/nginx/blobs/sha256:abc123def456", + wantName: "", + wantDigest: "", + }, + { + path: "library/nginx/blobs/sha512:" + strings.Repeat("a", 64), + wantName: "", + wantDigest: "", }, { path: "invalid/path", @@ -726,6 +741,7 @@ func TestContainerHandler_BlobHead_CacheHitSkipsUpstreamAndAuth(t *testing.T) { } req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil) + req = req.WithContext(ServingHead(req.Context())) w := httptest.NewRecorder() h.Routes().ServeHTTP(w, req) @@ -749,7 +765,7 @@ func TestContainerHandler_BlobHead_CacheHitSkipsUpstreamAndAuth(t *testing.T) { } } -func TestContainerHandler_BlobHead_DirectServeRedirects(t *testing.T) { +func TestContainerHandler_BlobHead_DirectServeAnswersFromCache(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") @@ -763,14 +779,21 @@ func TestContainerHandler_BlobHead_DirectServeRedirects(t *testing.T) { } req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil) + req = req.WithContext(ServingHead(req.Context())) w := httptest.NewRecorder() h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusFound { - t.Fatalf("status = %d, want %d", w.Code, http.StatusFound) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } - if got := w.Header().Get("Location"); got != store.signedURL { - t.Errorf("Location = %q, want %q", got, store.signedURL) + if got := w.Header().Get("Location"); got != "" { + t.Errorf("Location = %q, want empty", got) + } + if got := w.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) + } + if got := w.Header().Get("Content-Length"); got != "11" { + t.Errorf("Content-Length = %q, want %q", got, "11") } wantETag := `"` + sha256Hex("cached blob") + `"` if got := w.Header().Get("ETag"); got != wantETag { @@ -1476,3 +1499,52 @@ func TestContainerHandler_ManifestHonoursSeparateAcceptHeaders(t *testing.T) { t.Errorf("offline body = %q, want the cached manifest", w.Body.String()) } } + +func TestContainerHandler_BlobSHA512(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + const blob = "sha512 layer" + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader(blob)), + ContentType: "application/octet-stream", + } + sum := sha512.Sum512([]byte(blob)) + digest := "sha512:" + hex.EncodeToString(sum[:]) + h := &ContainerHandler{proxy: proxy, registryURL: "https://registry.example.test"} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/library/app/blobs/"+digest, nil)) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if w.Body.String() != blob { + t.Errorf("body = %q, want %q", w.Body.String(), blob) + } + if got := w.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) + } +} + +func TestContainerHandler_HeadManifestByDigestChecksUpstreamDigest(t *testing.T) { + requested := sha256Digest([]byte("requested")) + other := sha256Digest([]byte("other")) + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", ociManifestMediaType) + w.Header().Set("Docker-Content-Digest", other) + w.Header().Set("Content-Length", "5") + })) + defer registry.Close() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = registry.Client() + h := &ContainerHandler{proxy: proxy, registryURL: registry.URL} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/library/app/manifests/"+requested, nil)) + + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d", w.Code, http.StatusBadGateway) + } + if got := w.Header().Get("Docker-Content-Digest"); got == other { + t.Errorf("Docker-Content-Digest = %q, the upstream's other digest", got) + } +} diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index 5d95d16..b765f9a 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "slices" "strings" "time" ) @@ -16,6 +17,26 @@ import ( // through. type publishTimes map[string]time.Time +func (p *Proxy) lookupPublishTime(ctx context.Context, t timedMetadata, version string, listed func(metadataResult) ([]string, error)) (time.Time, bool, bool, error) { + var versions []string + docs, err := p.fetchTimedMetadata(ctx, t, func(doc metadataResult) ([]string, error) { + var err error + versions, err = listed(doc) + return versions, err + }) + switch { + case docs.doc.err != nil: + return time.Time{}, false, false, docs.doc.err + case docs.times.err != nil: + return time.Time{}, false, false, docs.times.err + case err != nil: + return time.Time{}, false, false, err + } + published, ok := docs.published[version] + withholdUnknown := !ok && docs.withholdUnknown && slices.Contains(versions, version) + return published, ok && !published.IsZero(), withholdUnknown, nil +} + // timedMetadata describes metadata whose publish times live in a second // document, as with the RubyGems compact index and the versions API, or a // Hex package and the Hex HTTP API. Both are read through the metadata cache. diff --git a/internal/handler/cran.go b/internal/handler/cran.go index 7cba9f4..10f1750 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -51,21 +51,41 @@ func (h *CRANHandler) Routes() http.Handler { mux.HandleFunc("GET /src/contrib/PACKAGES.gz", h.proxyCached) mux.HandleFunc("GET /src/contrib/PACKAGES.rds", h.proxyCached) + // Repository metadata such as archive.rds and current.rds, used by + // remotes::install_version and pak to find archived versions. + mux.HandleFunc("GET /src/contrib/Meta/{filename}", h.handleMeta) + // Binary package indexes mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES", h.proxyCached) mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES.gz", h.proxyCached) mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES.rds", h.proxyCached) + // macOS binaries for R >= 4.3 use two platform segments, e.g. + // bin/macosx/big-sur-arm64/contrib/4.4/. + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES", h.proxyCached) + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES.gz", h.proxyCached) + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES.rds", h.proxyCached) + // Source package downloads mux.HandleFunc("GET /src/contrib/{filename}", h.handleSourceDownload) mux.HandleFunc("GET /src/contrib/Archive/{name}/{filename}", h.handleSourceDownload) // Binary package downloads mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/{filename}", h.handleBinaryDownload) + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/{filename}", h.handleBinaryDownload) return rejectQueryChars(mux) } +// handleMeta serves the .rds files under src/contrib/Meta as cached metadata. +func (h *CRANHandler) handleMeta(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.PathValue("filename"), ".rds") { + http.NotFound(w, r) + return + } + h.proxyCached(w, r) +} + // handleSourceDownload serves a source package, fetching and caching from upstream. func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Request) { filename := r.PathValue("filename") @@ -99,6 +119,9 @@ func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Reques // handleBinaryDownload serves a binary package, fetching and caching from upstream. func (h *CRANHandler) handleBinaryDownload(w http.ResponseWriter, r *http.Request) { platform := r.PathValue("platform") + if arch := r.PathValue("arch"); arch != "" { + platform += "-" + arch + } rversion := r.PathValue("rversion") filename := r.PathValue("filename") diff --git a/internal/handler/cran_test.go b/internal/handler/cran_test.go index 6a9c151..6c156c5 100644 --- a/internal/handler/cran_test.go +++ b/internal/handler/cran_test.go @@ -1,8 +1,14 @@ package handler import ( + "io" "log/slog" + "net/http" + "net/http/httptest" + "strings" "testing" + + "github.com/git-pkgs/registries/fetch" ) func TestCRANParseSourceFilename(t *testing.T) { @@ -50,3 +56,68 @@ func TestCRANParseBinaryFilename(t *testing.T) { } } } + +func TestCRANHandler_MacOSTwoSegmentIndexesAndMeta(t *testing.T) { + for _, path := range []string{ + "/bin/macosx/big-sur-arm64/contrib/4.4/PACKAGES", + "/bin/macosx/big-sur-x86_64/contrib/4.4/PACKAGES.gz", + "/bin/macosx/sonoma-arm64/contrib/4.5/PACKAGES.rds", + "/src/contrib/Meta/archive.rds", + "/src/contrib/Meta/current.rds", + } { + t.Run(path, func(t *testing.T) { + assertUpstreamProxied(t, "index", path, + func(proxy *Proxy, upstreamURL string) http.Handler { + h := &CRANHandler{proxy: proxy, upstreamURL: upstreamURL, proxyURL: "http://localhost"} + return h.Routes() + }, + ) + }) + } +} + +func TestCRANHandler_MacOSTwoSegmentBinaryDownload(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("fetched binary")), + ContentType: "application/x-gzip", + } + + h := NewCRANHandler(proxy, "http://localhost") + h.upstreamURL = "https://cran.r-project.org" + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz") + if err != nil { + t.Fatalf("request failed: %v", err) + } + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + want := "https://cran.r-project.org/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz" + if fetcher.fetchedURL != want { + t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) + } +} + +func TestCRANHandler_MacOSTwoSegmentBinaryCacheIdentity(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackageWithPURL(t, db, store, "cran", "dplyr", "1.1.4_macosx-big-sur-arm64_4.4", "dplyr_1.1.4.tgz", "arm64 binary") + + h := NewCRANHandler(proxy, "http://localhost") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if string(body) != "arm64 binary" { + t.Errorf("body = %q, want %q", body, "arm64 binary") + } +} diff --git a/internal/handler/debian.go b/internal/handler/debian.go index b2419f6..cb68afd 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "net/http" + pathpkg "path" "regexp" "sort" "strings" @@ -108,10 +109,10 @@ func (h *DebianHandler) Routes() http.Handler { } } - // Route based on path type + // Route by filename rather than path prefix because repositories such as + // Docker's nest the pool under dists//. switch { - case strings.HasPrefix(rest, "pool/"): - // Package downloads - cache these + case isDebPackageFile(rest): h.handlePackageDownload(w, r, repository, upstreamURL, rest) case strings.HasPrefix(rest, "dists/"): // Repository metadata - served through the metadata cache @@ -123,6 +124,10 @@ func (h *DebianHandler) Routes() http.Handler { })) } +func isDebPackageFile(path string) bool { + return strings.HasSuffix(path, ".deb") +} + // repositoryNotFound answers a request addressed to a repository that is not // configured, naming the ones that are so a misspelling is self-evident. func (h *DebianHandler) repositoryNotFound(w http.ResponseWriter, name string) { @@ -164,7 +169,12 @@ func (h *DebianHandler) handlePackageDownload( filename := path[strings.LastIndex(path, "/")+1:] downloadURL := fmt.Sprintf("%s/%s", upstreamURL, path) - cacheFilename := h.artifactCacheFilename(repository, filename) + cacheFilename, err := h.proxy.cacheFilenameKeepingLegacy("deb", name, version, + h.artifactCacheFilename(repository, path), h.legacyArtifactCacheFilename(repository, filename), downloadURL) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch package") + return + } h.proxy.Logger.Info("debian package download", "repository", repository, @@ -183,19 +193,66 @@ func (h *DebianHandler) handlePackageDownload( // handleMetadata serves repository metadata files through the metadata cache, // so they keep resolving while the upstream archive is unreachable. +// Index files the cached InRelease or Release lists follow that listing (see +// proxyListedIndex). func (h *DebianHandler) handleMetadata( w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, ) { cacheKey := h.metadataCacheKeyFor(repository, upstreamURL, path) - h.proxy.ProxyCached(w, r, fmt.Sprintf("%s/%s", upstreamURL, path), "debian", cacheKey, "*/*") + fileURL := fmt.Sprintf("%s/%s", upstreamURL, path) + switch pathpkg.Base(path) { + case "InRelease", "Release", "Release.gpg": + h.proxy.ProxyCached(w, r, fileURL, "debian", cacheKey, "*/*") + return + } + release, want := h.releaseListing(r, repository, upstreamURL, path) + h.proxy.proxyListedIndex(w, r, fileURL, "debian", cacheKey, release, want) } -// artifactCacheFilename scopes a package by repository, since the same -// filename can hold different bytes in different archives. The main archive -// keeps the unscoped filename its existing cache entries were stored under. -func (h *DebianHandler) artifactCacheFilename(repository, filename string) string { +// releaseListing finds the cached release file of the suite path belongs +// to and the sha256 it lists for path, nearest directory first, skipping +// release files that list nothing for path (such as the legacy +// per-component Release). Where both InRelease and Release list it, the one +// fetched last wins: apt reads InRelease and debootstrap Release, and a +// client asking for the index has most likely just fetched one of them. +func (h *DebianHandler) releaseListing(r *http.Request, repository, upstreamURL, path string) (*cachedRelease, string) { + for dir := pathpkg.Dir(path); strings.HasPrefix(dir, "dists/"); dir = pathpkg.Dir(dir) { + var found *cachedRelease + var want string + for _, name := range []string{"InRelease", "Release"} { + release := h.proxy.readCachedRelease(r.Context(), "debian", h.metadataCacheKeyFor(repository, upstreamURL, dir+"/"+name)) + if release == nil { + continue + } + sum := debianReleaseSHA256(release.body, strings.TrimPrefix(path, dir+"/")) + if sum != "" && (found == nil || release.fetchedAt.After(found.fetchedAt)) { + found, want = release, sum + } + } + if found != nil { + return found, want + } + } + return nil, "" +} + +// artifactCacheFilename names a package in the cache by its repository and +// its path in it, since the same file name can hold different bytes in +// different archives, and in different directories of one archive: Docker's +// repository has a pool under each suite. +func (h *DebianHandler) artifactCacheFilename(repository, path string) string { + if repository == "" { + return path + } + return repository + "/" + path +} + +// legacyArtifactCacheFilename is the name packages were cached under before +// they were keyed by path: the file name, scoped by repository for a named +// one. It is still read for the URL an entry was fetched from. +func (h *DebianHandler) legacyArtifactCacheFilename(repository, filename string) string { if repository == "" { return filename } @@ -226,12 +283,7 @@ var debPackagePattern = regexp.MustCompile(`^(.+)_([^_]+)_([^_]+)\.deb$`) // parsePoolPath extracts package info from a pool path. // Example: pool/main/n/nginx/nginx_1.18.0-6_amd64.deb func (h *DebianHandler) parsePoolPath(path string) (name, version, arch string) { - // Get the filename - idx := strings.LastIndex(path, "/") - if idx < 0 { - return "", "", "" - } - filename := path[idx+1:] + filename := path[strings.LastIndex(path, "/")+1:] // Parse the filename matches := debPackagePattern.FindStringSubmatch(filename) diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index f31202f..f9c6284 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -1,10 +1,12 @@ package handler import ( + "context" "fmt" "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" @@ -25,6 +27,7 @@ func TestDebianHandler_parsePoolPath(t *testing.T) { "nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", "amd64", }, {"pool/main/o/openssl/openssl_3.0.2-0ubuntu1.15~build1_amd64.deb", "openssl", "3.0.2-0ubuntu1.15~build1", "amd64"}, + {"nginx_1.18.0-6_amd64.deb", "nginx", "1.18.0-6", "amd64"}, {"invalid/path", "", "", ""}, {"pool/main/n/nginx/nginx.deb", "", "", ""}, }) @@ -36,10 +39,11 @@ func TestDebianHandler_Routes(t *testing.T) { } // TestDebianHandler_LegacyCacheKeysUnchanged pins the artifact cache identity -// the main archive used before named repositories existed. Deployments carry -// warm caches across upgrades, so a changed key here silently discards them. -// Metadata is cheap to refetch; its keys are covered by -// TestMetadataCacheKeysUnambiguous. +// the main archive used before named repositories existed. Packages are now +// keyed by path, but deployments carry warm caches across upgrades, so +// entries under this name are still read for the URL they were fetched from +// (TestDebianHandler_LegacyPackageEntryServedForItsPath). Metadata is cheap +// to refetch; its keys are covered by TestMetadataCacheKeysUnambiguous. func TestDebianHandler_LegacyCacheKeysUnchanged(t *testing.T) { const ( poolPath = "pool/main/h/hello/hello_2.10-3_amd64.deb" @@ -52,12 +56,12 @@ func TestDebianHandler_LegacyCacheKeysUnchanged(t *testing.T) { // Artifact cache: the bare filename, with no repository prefix. filename := poolPath[strings.LastIndex(poolPath, "/")+1:] - if got := h.artifactCacheFilename("", filename); got != "hello_2.10-3_amd64.deb" { + if got := h.legacyArtifactCacheFilename("", filename); got != "hello_2.10-3_amd64.deb" { t.Errorf("legacy artifact cache filename = %q, want %q", got, "hello_2.10-3_amd64.deb") } // A named repository must not reuse either identity. - if got := h.artifactCacheFilename("security", filename); got == "hello_2.10-3_amd64.deb" { + if got := h.legacyArtifactCacheFilename("security", filename); got == "hello_2.10-3_amd64.deb" { t.Error("named repository reused the legacy artifact cache filename") } if h.metadataCacheKeyFor("security", "https://security.test", distPath) == h.metadataCacheKeyFor("", h.upstreamURL, distPath) { @@ -251,6 +255,44 @@ func TestDebianHandler_PackageCacheScopedPerRepository(t *testing.T) { } } +func TestDebianHandler_FlatArchivePackageIsCached(t *testing.T) { + const pkgBody = "docker-ce package bytes" + + var upstreamHits int + + archive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamHits++ + _, _ = fmt.Fprint(w, pkgBody) + })) + defer archive.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(archive.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewDebianHandler(proxy, "http://proxy.example", "https://archive.test", map[string]string{ + "docker": archive.URL, + }) + + for _, path := range []string{ + "/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb", + "/docker/foo_1.0_amd64.deb", + } { + for attempt := 1; attempt <= 2; attempt++ { + w := serveDebianRequest(h, path) + if w.Code != http.StatusOK || w.Body.String() != pkgBody { + t.Fatalf("%s request %d: status = %d, body = %q, want 200 %q", + path, attempt, w.Code, w.Body.String(), pkgBody) + } + } + } + + if upstreamHits != 2 { + t.Errorf("upstream fetched %d times for four requests, want 2: the package is not cached", upstreamHits) + } +} + func serveDebianRequest(h *DebianHandler, target string) *httptest.ResponseRecorder { w := httptest.NewRecorder() h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) @@ -285,3 +327,72 @@ func TestDebRetentionKey(t *testing.T) { t.Error("debRetentionKey(nil) accepted") } } + +// debianSuiteArchive serves a different build of one package file name per +// suite, as Docker's repository nests a pool under each suite. +func debianSuiteArchive(t *testing.T, available *atomic.Bool) *httptest.Server { + t.Helper() + archive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + suite := strings.Split(strings.TrimPrefix(r.URL.Path, "/dists/"), "/")[0] + _, _ = fmt.Fprint(w, suite+" build") + })) + t.Cleanup(archive.Close) + return archive +} + +const ( + debianBullseyePackage = "/docker/dists/bullseye/pool/stable/amd64/containerd.io_1.7.25-1_amd64.deb" + debianBookwormPackage = "/docker/dists/bookworm/pool/stable/amd64/containerd.io_1.7.25-1_amd64.deb" +) + +// TestDebianHandler_PackageCacheScopedPerPath checks that one file name in +// two directories of a repository does not resolve to one cached artifact. +func TestDebianHandler_PackageCacheScopedPerPath(t *testing.T) { + var available atomic.Bool + available.Store(true) + archive := debianSuiteArchive(t, &available) + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(archive.Client()), fetch.WithMaxRetries(0)) + h := NewDebianHandler(proxy, "http://proxy.example", "https://archive.test", map[string]string{"docker": archive.URL}) + + for _, tc := range []struct{ path, want string }{ + {debianBullseyePackage, "bullseye build"}, + {debianBookwormPackage, "bookworm build"}, + } { + if w := serveDebianRequest(h, tc.path); w.Code != http.StatusOK || w.Body.String() != tc.want { + t.Errorf("GET %s = %d %q, want %q", tc.path, w.Code, w.Body.String(), tc.want) + } + } +} + +// TestDebianHandler_LegacyPackageEntryServedForItsPath checks that a package +// cached under the file name alone, as before packages were keyed by path, is +// still served for the path it was fetched from, and only for that path. +func TestDebianHandler_LegacyPackageEntryServedForItsPath(t *testing.T) { + var available atomic.Bool + available.Store(true) + archive := debianSuiteArchive(t, &available) + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(archive.Client()), fetch.WithMaxRetries(0)) + h := NewDebianHandler(proxy, "http://proxy.example", "https://archive.test", map[string]string{"docker": archive.URL}) + + // The entry an earlier version cached for the bullseye path. + legacy, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "deb", "containerd.io", "1.7.25-1", + "docker/containerd.io_1.7.25-1_amd64.deb", archive.URL+strings.TrimPrefix(debianBullseyePackage, "/docker")) + if err != nil { + t.Fatalf("seed legacy entry: %v", err) + } + _ = legacy.Reader.Close() + + if w := serveDebianRequest(h, debianBookwormPackage); w.Body.String() != "bookworm build" { + t.Errorf("bookworm = %d %q, want its own build, not the legacy bullseye entry", w.Code, w.Body.String()) + } + available.Store(false) + if w := serveDebianRequest(h, debianBullseyePackage); w.Code != http.StatusOK || w.Body.String() != "bullseye build" { + t.Errorf("bullseye with the archive down = %d %q, want the legacy entry", w.Code, w.Body.String()) + } +} diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go new file mode 100644 index 0000000..4ede9a8 --- /dev/null +++ b/internal/handler/download_cooldown.go @@ -0,0 +1,159 @@ +package handler + +import ( + "net/http" + "sync" + "time" + + "golang.org/x/sync/singleflight" +) + +// publishTimeLookup reads a version's publish time from upstream metadata. ok +// is false when the metadata has no usable time for the version. withholdUnknown +// marks a listed version missing from stale publish-time metadata. +type publishTimeLookup func() (published time.Time, ok, withholdUnknown bool, err error) + +// downloadVersion names the version an artifact download is for, so its +// cooldown can be checked before the artifact is served. +type downloadVersion struct { + ecosystem string + // policyName is the package name the cooldown policy is keyed by, + // canonicalPackagePURL(ecosystem, policyName). + policyName string + // storedName and version name the versions row the publish time is kept + // in, the same names the artifact is cached under. An empty storedName + // keeps no time, and every check asks lookup. + storedName string + version string + cacheName string + filename string + lookup publishTimeLookup +} + +// withholdInCooldown reports whether a download is inside the cooldown +// window, or a lookup failure prevents an uncached download. +func (p *Proxy) withholdInCooldown(w http.ResponseWriter, r *http.Request, d downloadVersion) bool { + withheld, err := p.versionInCooldown(d) + if err != nil { + if !p.serveCachedDespiteCooldownError(w, r, d.ecosystem, d.cacheName, d.version, d.filename, err) { + p.writeCooldownLookupError(w, d.ecosystem, d.policyName, err) + } + return true + } + if !withheld { + return false + } + p.Logger.Info("cooldown: withholding "+d.ecosystem+" download", + "name", d.policyName, "version", d.version) + http.Error(w, ErrVersionNotFound.Error(), http.StatusNotFound) + return true +} + +func (p *Proxy) versionInCooldown(d downloadVersion) (bool, error) { + policyKey := canonicalPackagePURL(d.ecosystem, d.policyName) + if !p.cooldownApplies(d.ecosystem, d.policyName) { + return false, nil + } + + var versionPURL string + if d.storedName != "" { + versionPURL = canonicalVersionPURL(d.ecosystem, d.storedName, d.version) + if ver, err := p.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return !p.cooldown().IsAllowed(d.ecosystem, policyKey, ver.PublishedAt.Time), nil + } + } + + published, ok, withholdUnknown, err := d.lookup() + if err != nil { + return false, err + } + if !ok { + return withholdUnknown, nil + } + + if versionPURL != "" { + if err := p.DB.SetVersionPublishedAt(versionPURL, canonicalPackagePURL(d.ecosystem, d.storedName), published); err != nil { + p.Logger.Warn("cooldown: could not store publish time", + "ecosystem", d.ecosystem, "name", d.policyName, "version", d.version, "error", err) + } + } + + return !p.cooldown().IsAllowed(d.ecosystem, policyKey, published), nil +} + +// maxPublishIndexes bounds how many documents publishIndexCache keeps an +// index for. +const maxPublishIndexes = 64 + +// publishIndexCache keeps, for each large metadata document, the publish +// times a download check reads from it, so a document is decompressed and +// parsed once per version from upstream rather than on every download. +// Concurrent checks for a document not indexed yet share one parse. +type publishIndexCache struct { + mu sync.Mutex + entries map[string]publishIndexEntry + group singleflight.Group +} + +type publishIndexEntry struct { + digest string + index any + expires time.Time +} + +// publishIndex reuses parsed documents by digest, or for a minute without one. +func publishIndex[V any](p *Proxy, key string, load func() metadataResult, build func(metadataResult) (V, error)) (V, error) { //nolint:ireturn // Supports timestamp maps and Helm lookups. + c := &p.sharedProxy().publishIndexes + fresh := func() (V, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if e, ok := c.entries[key]; ok && !e.expires.IsZero() && p.clock().Before(e.expires) { + return e.index.(V), true + } + var zero V + return zero, false + } + if index, ok := fresh(); ok { + return index, nil + } + v, err, _ := c.group.Do(key, func() (any, error) { + if index, ok := fresh(); ok { + return index, nil + } + res := load() + if res.err == nil && res.digest != "" { + c.mu.Lock() + entry, ok := c.entries[key] + c.mu.Unlock() + if ok && entry.digest == res.digest { + return entry.index, nil + } + } + index, err := build(res) + if err != nil { + return nil, err + } + c.mu.Lock() + defer c.mu.Unlock() + if c.entries == nil { + c.entries = make(map[string]publishIndexEntry) + } + if _, ok := c.entries[key]; !ok && len(c.entries) >= maxPublishIndexes { + for evict := range c.entries { + delete(c.entries, evict) + break + } + } + entry := publishIndexEntry{digest: res.digest, index: index} + if res.digest == "" || !p.CacheMetadata { + entry.expires = p.clock().Add(time.Minute) + } + c.entries[key] = entry + return index, nil + }) + if err != nil { + var zero V + return zero, err + } + return v.(V), nil +} diff --git a/internal/handler/download_cooldown_cache_test.go b/internal/handler/download_cooldown_cache_test.go new file mode 100644 index 0000000..d51b862 --- /dev/null +++ b/internal/handler/download_cooldown_cache_test.go @@ -0,0 +1,115 @@ +package handler + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestGemDownloadSharesPublishTimeMetadata(t *testing.T) { + u, proxy, h := gemCacheSetup(t) + wantStatus(t, serveGet(t, h, gemInfoPath), http.StatusOK, "") + seedPackage(t, proxy.DB, proxy.Storage.(*mockStorage), "gem", "testgem", "1.0.0", "testgem-1.0.0.gem", "gem bytes") + wantStatus(t, serveGet(t, h, "/gems/testgem-1.0.0.gem"), http.StatusOK, "") + if n, _ := u.counts(gemVersionsPath); n != 1 { + t.Fatalf("versions API fetched %d times, want 1", n) + } +} + +type downloadPublishTimeCase struct { + name string + down bool + absent bool + untimed bool + refresh bool + wantStatus int +} + +func TestDownloadCooldownRefreshesMissingPublishTimes(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.ecosystem != "gem" && c.ecosystem != "hex" { + continue + } + for _, scenario := range []downloadPublishTimeCase{ + {name: "refreshed times", refresh: true, wantStatus: http.StatusNotFound}, + {name: "stale times and listed version", down: true, wantStatus: http.StatusNotFound}, + {name: "stale times and absent version", down: true, absent: true, wantStatus: http.StatusOK}, + {name: "fresh times without version", wantStatus: http.StatusOK}, + {name: "stale times with untimed version", down: true, untimed: true, wantStatus: http.StatusOK}, + } { + for _, cached := range []bool{false, true} { + if !cached && scenario.wantStatus == http.StatusOK { + continue + } + t.Run(fmt.Sprintf("%s/%s/cached=%t", c.ecosystem, scenario.name, cached), func(t *testing.T) { + assertDownloadPublishTime(t, c, scenario, cached) + }) + } + } + } +} + +func assertDownloadPublishTime(t *testing.T, c downloadCooldownCase, scenario downloadPublishTimeCase, cached bool) { + t.Helper() + u := newCooldownCountingUpstream(t) + proxy := cooldownCacheProxy(t, u) + fetcher := proxy.Fetcher.(*mockFetcher) + fetcher.fetchErr = errors.New("unchecked artifact download") + h := c.handler(proxy, u.URL) + docPath, timesPath := "/info/demo", "/api/v1/versions/demo.json" + if c.ecosystem == "hex" { + docPath, timesPath = "/packages/demo", "/api/packages/demo" + } + oldTimes := map[string]time.Time{cooldownOldVersion: time.Now().Add(-30 * 24 * time.Hour)} + if scenario.untimed { + oldTimes[cooldownRecentVersion] = time.Time{} + } + setDownloadMetadata(t, u, c, docPath, oldTimes) + setDownloadMetadata(t, u, c, timesPath, oldTimes) + wantStatus(t, serveGet(t, h, docPath), http.StatusOK, "") + + newTimes := map[string]time.Time{ + cooldownOldVersion: oldTimes[cooldownOldVersion], + cooldownUntimed: time.Now().Add(-time.Minute), + } + if !scenario.absent { + newTimes[cooldownRecentVersion] = time.Now().Add(-time.Hour) + } + setDownloadMetadata(t, u, c, docPath, newTimes) + expireCachedMetadata(t, proxy, c.ecosystem, docPath[1:]) + if scenario.refresh { + setDownloadMetadata(t, u, c, timesPath, newTimes) + } + u.setDown(timesPath, scenario.down) + if cached { + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + } + wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), scenario.wantStatus, "") + if fetcher.fetchCalled { + t.Error("fetched an unchecked artifact") + } + if n, _ := u.counts(timesPath); n != 2 { + t.Errorf("publish times fetched %d times, want 2", n) + } +} + +func setDownloadMetadata(t *testing.T, u *cooldownCountingUpstream, c downloadCooldownCase, path string, times map[string]time.Time) { + t.Helper() + w := httptest.NewRecorder() + c.metadata(w, httptest.NewRequest(http.MethodGet, path, nil), times) + wantStatus(t, w, http.StatusOK, "") + u.set(path, w.Body.Bytes()) +} + +func TestHexDownloadSharesPublishTimeMetadata(t *testing.T) { + u, proxy, h, _ := hexCacheSetup(t, 0) + wantStatus(t, serveGet(t, h, hexPackagePath), http.StatusOK, "") + seedPackage(t, proxy.DB, proxy.Storage.(*mockStorage), "hex", "testpkg", "1.0.0", "testpkg-1.0.0.tar", "hex bytes") + wantStatus(t, serveGet(t, h, "/tarballs/testpkg-1.0.0.tar"), http.StatusOK, "") + if n, _ := u.counts(hexAPIPath); n != 1 { + t.Fatalf("Hex API fetched %d times, want 1", n) + } +} diff --git a/internal/handler/download_cooldown_errors_test.go b/internal/handler/download_cooldown_errors_test.go new file mode 100644 index 0000000..b214db2 --- /dev/null +++ b/internal/handler/download_cooldown_errors_test.go @@ -0,0 +1,101 @@ +package handler + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" +) + +func TestDownloadCooldownLookupFailures(t *testing.T) { + for _, c := range downloadCooldownCases() { + for _, failure := range []string{"upstream error", "invalid metadata", "rate limited"} { + for _, cached := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/cached=%t", c.ecosystem, failure, cached), func(t *testing.T) { + assertDownloadLookupFailure(t, c, failure, cached) + }) + } + } + } +} + +func assertDownloadLookupFailure(t *testing.T, c downloadCooldownCase, failure string, cached bool) { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if failure == "upstream error" { + w.WriteHeader(http.StatusInternalServerError) + } + _, _ = w.Write([]byte("invalid metadata")) + })) + t.Cleanup(upstream.Close) + proxy, _, _, fetcher := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + proxy.HTTPClient = upstream.Client() + if failure == "rate limited" { + proxy.HTTPClient = &http.Client{Transport: roundTripperFunc(func(_ *http.Request) (*http.Response, error) { + return nil, ®istryclient.HeldError{Host: "upstream.example", Until: time.Now().Add(time.Minute), RetryAfter: time.Minute} + })} + } + if cached { + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + } + w := serveGet(t, c.handler(proxy, upstream.URL), c.path(cooldownRecentVersion)) + want := http.StatusBadGateway + if failure == "rate limited" { + want = http.StatusServiceUnavailable + } + if cached { + want = http.StatusOK + if w.Body.Len() == 0 { + t.Fatal("cached artifact body is empty") + } + } + wantStatus(t, w, want, "") + if fetcher.fetchCalled { + t.Fatal("lookup failure fetched an unchecked artifact") + } +} + +func TestDownloadPublishIndexesHaveShortTTLWithoutMetadataCache(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.ecosystem != "conda" && c.ecosystem != "helm" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + var calls atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + c.metadata(w, r, map[string]time.Time{ + cooldownOldVersion: time.Now().Add(-30 * 24 * time.Hour), + cooldownRecentVersion: time.Now().Add(-time.Hour), + }) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + clock := &testClock{t: time.Now()} + proxy.now = clock.now + c.seedCachedArtifact(t, proxy, cooldownOldVersion) + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + h := c.handler(proxy, upstream.URL) + for range 3 { + wantStatus(t, serveGet(t, h, c.path(cooldownOldVersion)), http.StatusOK, "") + wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), http.StatusNotFound, "") + } + if n := calls.Load(); n != 1 { + t.Fatalf("index downloads within TTL = %d, want 1", n) + } + clock.advance(time.Minute + time.Second) + wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), http.StatusNotFound, "") + if n := calls.Load(); n != 2 { + t.Errorf("index downloads after expiry = %d, want 2", n) + } + }) + } +} diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go new file mode 100644 index 0000000..1fd915f --- /dev/null +++ b/internal/handler/download_cooldown_test.go @@ -0,0 +1,483 @@ +package handler + +import ( + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// Versions every download cooldown fixture lists: one published before the +// window, one inside it, and one without a publish time. +const ( + cooldownOldVersion = "1.0.0" + cooldownRecentVersion = "2.0.0" + cooldownUntimed = "3.0.0" +) + +// downloadCooldownCase drives one ecosystem's artifact endpoint. The package +// is always named demo, so the policy key is pkg:{ecosystem}/demo. +type downloadCooldownCase struct { + ecosystem string + // cachedName is the name the artifact is cached under, and keepsTime + // whether the publish time is kept in its versions row. + cachedName string + keepsTime bool + // artifactVersion maps a release to the version the artifact is cached + // under, and filename and path to its file and download path. + artifactVersion func(version string) string + filename func(version string) string + path func(version string) string + // metadata serves the upstream document holding the publish times; a + // zero time is listed without one. + metadata func(w http.ResponseWriter, r *http.Request, times map[string]time.Time) + handler func(proxy *Proxy, upstreamURL string) http.Handler +} + +func sameVersion(version string) string { return version } + +func chartContent(version string) string { return "chart " + version } + +func downloadCooldownCases() []downloadCooldownCase { + return []downloadCooldownCase{ + { + ecosystem: "cargo", + cachedName: "demo", + keepsTime: true, + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".crate" }, + path: func(v string) string { return "/crates/demo/" + v + "/download" }, + metadata: cargoCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewCargoHandler(p, "http://proxy", u, u).Routes() }, + }, + { + ecosystem: "gem", + cachedName: "demo", + keepsTime: true, + // Platform gems carry the platform in the filename and the + // cached version, but not in the versions API number. + artifactVersion: func(v string) string { return v + "-x86_64-linux" }, + filename: func(v string) string { return "demo-" + v + "-x86_64-linux.gem" }, + path: func(v string) string { return "/gems/demo-" + v + "-x86_64-linux.gem" }, + metadata: gemCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewGemHandlerWithUpstream(p, "http://proxy", u).Routes() }, + }, + { + ecosystem: "hex", + cachedName: "demo", + keepsTime: true, + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".tar" }, + path: func(v string) string { return "/tarballs/demo-" + v + ".tar" }, + metadata: hexCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewHexHandlerWithUpstreams(p, "http://proxy", u, u).Routes() + }, + }, + { + ecosystem: "pub", + cachedName: "demo", + keepsTime: true, + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".tar.gz" }, + path: func(v string) string { return "/packages/demo/versions/" + v + ".tar.gz" }, + metadata: pubCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewPubHandlerWithUpstream(p, "http://proxy", u).Routes() }, + }, + { + // Conda caches under {channel}/{name}, but the policy is keyed by + // the name alone, as the repodata filter keys it. Each build has + // its own timestamp, so none is kept in the version's row. + ecosystem: "conda", + cachedName: "conda-forge/demo", + artifactVersion: sameVersion, + filename: func(v string) string { return "noarch/demo-" + v + "-py_0.conda" }, + path: func(v string) string { return "/conda-forge/noarch/demo-" + v + "-py_0.conda" }, + metadata: condaCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewCondaHandlerWithUpstream(p, "http://proxy", u).Routes() + }, + }, + { + // Helm caches charts under the repository and digest, so the + // chart name and time come from the index on every check. + cachedName: "stable", + ecosystem: "helm", + artifactVersion: func(v string) string { return sha256Hex(chartContent(v)) }, + filename: func(v string) string { return "demo-" + v + ".tgz" }, + path: func(v string) string { return "/stable/charts/" + sha256Hex(chartContent(v)) + "/demo-" + v + ".tgz" }, + metadata: helmCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewHelmHandler(p, "http://proxy", map[string]string{"stable": u + "/charts"}).Routes() + }, + }, + } +} + +func cargoCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/de/mo/demo" { + http.NotFound(w, r) + return + } + for version, published := range times { + line := map[string]string{"name": "demo", "vers": version} + if !published.IsZero() { + line["pubtime"] = published.Format(time.RFC3339) + } + encoded, _ := json.Marshal(line) + _, _ = fmt.Fprintf(w, "%s\n", encoded) + } +} + +func gemCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path == "/info/demo" { + _, _ = fmt.Fprintln(w, "---") + for version := range times { + _, _ = fmt.Fprintf(w, "%s-x86_64-linux |checksum:a\n", version) + } + return + } + if r.URL.Path != "/api/v1/versions/demo.json" { + http.NotFound(w, r) + return + } + var versions []gemVersion + for version, published := range times { + v := gemVersion{Number: version, Platform: "x86_64-linux"} + if !published.IsZero() { + v.CreatedAt = published.Format(time.RFC3339) + } + versions = append(versions, v) + } + _ = json.NewEncoder(w).Encode(versions) +} + +func hexCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path == "/packages/demo" { + var versions []string + for version := range times { + versions = append(versions, version) + } + _, _ = w.Write(buildHexSigned(buildHexPackage("demo", versions))) + return + } + if r.URL.Path != "/api/packages/demo" { + http.NotFound(w, r) + return + } + var pkg hexPackageAPI + for version, published := range times { + release := hexRelease{Version: version} + if !published.IsZero() { + release.InsertedAt = published.Format(time.RFC3339Nano) + } + pkg.Releases = append(pkg.Releases, release) + } + _ = json.NewEncoder(w).Encode(pkg) +} + +func pubCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/api/packages/demo" { + http.NotFound(w, r) + return + } + var versions []map[string]string + for version, published := range times { + v := map[string]string{"version": version} + if !published.IsZero() { + v["published"] = published.Format(time.RFC3339) + } + versions = append(versions, v) + } + _ = json.NewEncoder(w).Encode(map[string]any{"name": "demo", "versions": versions}) +} + +func condaCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/conda-forge/noarch/repodata.json" { + http.NotFound(w, r) + return + } + packages := map[string]map[string]any{} + for version, published := range times { + entry := map[string]any{"name": "demo", "version": version} + if !published.IsZero() { + entry["timestamp"] = published.UnixMilli() + } + packages["demo-"+version+"-py_0.conda"] = entry + } + // The download check asks for gzip, as proxyCached does. + w.Header().Set(headerContentEncoding, "gzip") + gz := gzip.NewWriter(w) + _ = json.NewEncoder(gz).Encode(map[string]any{"packages.conda": packages}) + _ = gz.Close() +} + +func helmCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/charts/index.yaml" { + http.NotFound(w, r) + return + } + _, _ = fmt.Fprintln(w, "apiVersion: v1\nentries:\n demo:") + for version, published := range times { + _, _ = fmt.Fprintf(w, " - digest: %s\n urls: [demo-%s.tgz]\n version: %s\n", + sha256Hex(chartContent(version)), version, version) + if !published.IsZero() { + _, _ = fmt.Fprintf(w, " created: %s\n", published.Format(time.RFC3339)) + } + } +} + +// seedCachedArtifact caches the release's artifact the way an earlier, +// cooldown-free download would have. +func (c downloadCooldownCase) seedCachedArtifact(t *testing.T, proxy *Proxy, version string) { + t.Helper() + db, store := proxy.DB, proxy.Storage.(*mockStorage) + content := "artifact " + version + if c.ecosystem == "helm" { + content = chartContent(version) + } + seedPackage(t, db, store, c.ecosystem, c.cachedName, c.artifactVersion(version), c.filename(version), content) +} + +// TestDownloadCooldownWithholdsCachedArtifacts requests each ecosystem's +// artifact endpoint for a version that was cached before the cooldown was +// raised. A version inside the window must be withheld, from the cache too; +// one past the window, or without a known publish time, is served. +func TestDownloadCooldownWithholdsCachedArtifacts(t *testing.T) { + now := time.Now().Truncate(time.Second) + times := map[string]time.Time{ + cooldownOldVersion: now.Add(-30 * 24 * time.Hour), + cooldownRecentVersion: now.Add(-time.Hour), + cooldownUntimed: {}, + } + statuses := map[string]int{ + cooldownOldVersion: http.StatusOK, + cooldownRecentVersion: http.StatusNotFound, + cooldownUntimed: http.StatusOK, + } + + for _, c := range downloadCooldownCases() { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.metadata(w, r, times) + })) + t.Cleanup(upstream.Close) + + for _, version := range []string{cooldownOldVersion, cooldownRecentVersion, cooldownUntimed} { + t.Run(c.ecosystem+"/"+version, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, version) + proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ + canonicalPackagePURL(c.ecosystem, "demo"): "7d", + }} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(version), nil)) + + if w.Code != statuses[version] { + t.Fatalf("status = %d, want %d (body %q)", w.Code, statuses[version], w.Body.String()) + } + if w.Code == http.StatusOK && w.Body.String() == "" { + t.Error("served an empty body") + } + if !c.keepsTime || times[version].IsZero() { + return + } + ver, err := db.GetVersionByPURL(canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(version))) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("publish time not stored: %+v, %v", ver, err) + } + if got, want := ver.PublishedAt.Time, times[version]; !got.Equal(want) { + t.Errorf("stored publish time = %v, want %v", got, want) + } + }) + } + } +} + +// TestDownloadCooldownUsesStoredPublishTime checks that a stored publish +// time decides the download without asking the upstream, which here fails. +func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(upstream.Close) + + for _, c := range downloadCooldownCases() { + if !c.keepsTime { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + versionPURL := canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(cooldownRecentVersion)) + packagePURL := canonicalPackagePURL(c.ecosystem, c.cachedName) + if err := db.SetVersionPublishedAt(versionPURL, packagePURL, time.Now().Add(-time.Hour)); err != nil { + t.Fatalf("seeding publish time: %v", err) + } + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 (body %q)", w.Code, w.Body.String()) + } + }) + } +} + +// TestDownloadCooldownOtherPackagesSkipLookup checks that a download outside +// every cooldown rule is served without reading upstream metadata. Helm +// needs the index to learn the chart name, so it is left out. +func TestDownloadCooldownOtherPackagesSkipLookup(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.ecosystem == "helm" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected upstream request %s", r.URL.Path) + http.Error(w, "unexpected", http.StatusInternalServerError) + })) + t.Cleanup(upstream.Close) + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ + canonicalPackagePURL(c.ecosystem, "other"): "7d", + }} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (body %q)", w.Code, strings.TrimSpace(w.Body.String())) + } + }) + } +} + +// TestCondaDownloadCooldownPerBuild checks that builds of one version are +// judged by their own timestamps, whichever is requested first: a rebuild +// inside the window is withheld and an old build stays available. +func TestCondaDownloadCooldownPerBuild(t *testing.T) { + now := time.Now() + repodata := map[string]any{"packages.conda": map[string]any{ + "demo-1.0.0-py_0.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-400 * 24 * time.Hour).UnixMilli()}, + "demo-1.0.0-py_1.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-time.Hour).UnixMilli()}, + }} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(repodata) + })) + t.Cleanup(upstream.Close) + + builds := map[string]int{"py_0": http.StatusOK, "py_1": http.StatusNotFound} + for _, order := range [][]string{{"py_0", "py_1", "py_0"}, {"py_1", "py_0", "py_1"}} { + t.Run(strings.Join(order, ","), func(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + for build := range builds { + filename := "noarch/demo-1.0.0-" + build + ".conda" + seedPackage(t, db, store, "conda", "conda-forge/demo", "1.0.0", filename, "artifact "+build) + } + proxy.Cooldown = &cooldown.Config{Default: "7d"} + h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes() + + for _, build := range order { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/noarch/demo-1.0.0-"+build+".conda", nil)) + if w.Code != builds[build] { + t.Fatalf("%s status = %d, want %d", build, w.Code, builds[build]) + } + } + }) + } +} + +// TestPublishIndexBuildsOncePerDocument checks that an index is reused for +// the same document, rebuilt for a new one, and kept briefly without a digest. +func TestPublishIndexBuildsOncePerDocument(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + builds := 0 + build := func(res metadataResult) (map[string]bool, error) { + builds++ + return map[string]bool{string(res.body): true}, nil + } + steps := []struct { + res metadataResult + want string + wantBuilds int + }{ + {metadataResult{body: []byte("a"), digest: "d1"}, "a", 1}, + {metadataResult{body: []byte("ignored"), digest: "d1"}, "a", 1}, + {metadataResult{body: []byte("b"), digest: "d2"}, "b", 2}, + {metadataResult{body: []byte("c")}, "c", 3}, + {metadataResult{body: []byte("c")}, "c", 3}, + } + for i, step := range steps { + got, err := publishIndex(proxy, "key", func() metadataResult { return step.res }, build) + if err != nil || !got[step.want] || builds != step.wantBuilds { + t.Fatalf("step %d: publishIndex = %v, %v after %d builds, want %q after %d", i, got, err, builds, step.want, step.wantBuilds) + } + } +} + +// TestCondaDownloadCooldownPastMetadataLimit checks that a cached package is +// withheld when its repodata is over metadata_max_size, as the repodata +// endpoint hides it: once only decompressed, and once already compressed. +func TestCondaDownloadCooldownPastMetadataLimit(t *testing.T) { + plain := largeRepodata(500) + compressed := gzipPayload(t, plain) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(headerContentEncoding, "gzip") + _, _ = w.Write(compressed) + })) + t.Cleanup(upstream.Close) + + for _, tt := range []struct { + name string + limit int64 + }{ + {"decompressed over the limit", int64(len(compressed)) + 1}, + {"compressed over the limit", int64(len(compressed)) - 1}, + } { + t.Run(tt.name, func(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.MetadataMaxSize = tt.limit + fetcher.fetchErr = errors.New("artifact must be served from the cache") + for _, file := range []string{"pkg-0-old.conda", "pkg-1-new.conda"} { + seedPackage(t, db, store, "conda", "conda-forge/pkg", strings.Split(file, "-")[1], "linux-64/"+file, "artifact "+file) + } + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes() + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) + if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "pkg-1-new.conda") { + t.Fatalf("repodata status = %d, want 200 without pkg-1-new.conda", w.Code) + } + for file, want := range map[string]int{"pkg-0-old.conda": http.StatusOK, "pkg-1-new.conda": http.StatusNotFound} { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+file, nil)) + if w.Code != want { + t.Errorf("%s status = %d, want %d (body %q)", file, w.Code, want, w.Body.String()) + } + } + }) + } +} diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 216a279..c53585e 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -1,7 +1,9 @@ package handler import ( + "crypto/sha1" "database/sql" + "encoding/hex" "fmt" "io" "net/http" @@ -1074,8 +1076,7 @@ func TestMavenHandler_GradlePluginMarkerFallbackAndCache(t *testing.T) { func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) { paths := map[string]string{ - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "", - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml.sha1": "sha1", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "", } primaryHits := map[string]int{} @@ -1131,6 +1132,18 @@ func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) { t.Fatalf("GET %s did not hit plugin portal fallback", reqPath) } } + + // The checksum is computed from the metadata served from the fallback. + resp, err := http.Get(srv.URL + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml.sha1") + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + sum := sha1.Sum([]byte("")) //nolint:gosec // Maven publishes SHA-1 checksums. + if resp.StatusCode != http.StatusOK || string(body) != hex.EncodeToString(sum[:]) { + t.Fatalf("GET maven-metadata.xml.sha1: %d %q, want %x", resp.StatusCode, body, sum) + } } func TestMavenHandler_GradlePluginChecksumFallbackToPluginPortal(t *testing.T) { diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go index 153c0e6..c37d07d 100644 --- a/internal/handler/filename_download.go +++ b/internal/handler/filename_download.go @@ -3,6 +3,7 @@ package handler import ( "net/http" "strings" + "time" ) type filenameDownload struct { @@ -12,6 +13,9 @@ type filenameDownload struct { parseErr string fetchErr string parse func(string) (name, version string) + // publishTime reads a version's publish time from upstream metadata for + // the download-time cooldown check. + publishTime func(r *http.Request, name, version string) (time.Time, bool, bool, error) } func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d filenameDownload) { @@ -30,6 +34,13 @@ func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d p.Logger.Debug(d.ecosystem+" download request", "name", name, "version", version, "filename", filename) + if p.withholdInCooldown(w, r, downloadVersion{ + ecosystem: d.ecosystem, policyName: name, storedName: name, version: version, cacheName: name, filename: filename, + lookup: func() (time.Time, bool, bool, error) { return d.publishTime(r, name, version) }, + }) { + return + } + downloadURL := d.upstreamURL + r.URL.Path result, err := p.GetOrFetchArtifactFromURL( r.Context(), d.ecosystem, name, version, filename, downloadURL, diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 6e57559..fa7c24b 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -82,6 +82,7 @@ func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { parseErr: "could not parse gem filename", fetchErr: "failed to fetch gem", parse: h.parseGemFilename, + publishTime: h.publishTime, }) } @@ -304,6 +305,10 @@ func parseGemVersions(body []byte) (publishTimes, error) { return times, nil } +func (h *GemHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, bool, error) { + return h.proxy.lookupPublishTime(r.Context(), h.timedIndex(name), version, h.indexVersions) +} + // proxyCached forwards a metadata request with caching. func (h *GemHandler) proxyCached(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path diff --git a/internal/handler/git_mirror.go b/internal/handler/git_mirror.go new file mode 100644 index 0000000..38be233 --- /dev/null +++ b/internal/handler/git_mirror.go @@ -0,0 +1,149 @@ +package handler + +import ( + "context" + "fmt" + "log/slog" + "net/url" + "path" + "strings" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +// maxSubmoduleDepth bounds how deeply Mirror follows submodules of +// submodules. +const maxSubmoduleDepth = 8 + +// Mirror caches the repository at location, a host and path such as +// github.com/acme/widgets as a pkg:git PURL names it, from the upstream.git +// host serving it. With version, a full commit ID or a tag, it caches that +// commit too, and the commits its submodules pin, from the upstream.git +// hosts serving them. A submodule no upstream.git host serves is skipped. +// Clones through /git are then served from those copies. +func (h *GitHandler) Mirror(ctx context.Context, location, version string) (gitcache.MirrorResult, error) { + return h.mirror(ctx, location, version, map[string]bool{}, 0) +} + +func (h *GitHandler) mirror(ctx context.Context, location, version string, seen map[string]bool, depth int) (gitcache.MirrorResult, error) { + repo, err := h.mirrorRepository(location) + if err != nil { + return gitcache.MirrorResult{}, err + } + result, err := h.cache.Mirror(ctx, repo.key, repo.url, version, repo.upstream.fetchWants) + if err != nil { + return result, err + } + for _, sub := range result.Submodules { + subLocation, err := gitSubmoduleLocation(repo.url, sub.URL) + if err == nil { + _, err = h.mirrorRepository(subLocation) + } + if err != nil { + h.logger().Warn("git mirror: skipping submodule", "repository", location, "submodule", sub.Path, "url", sub.URL, "error", err) + continue + } + if seen[subLocation+"@"+sub.Commit] { + continue + } + seen[subLocation+"@"+sub.Commit] = true + if depth >= maxSubmoduleDepth { + return result, fmt.Errorf("submodule %s: nested more than %d deep", sub.Path, maxSubmoduleDepth) + } + subResult, err := h.mirror(ctx, subLocation, sub.Commit, seen, depth+1) + if err != nil { + return result, fmt.Errorf("submodule %s: %w", sub.Path, err) + } + result.Cached = result.Cached && subResult.Cached + result.Bytes += subResult.Bytes + } + return result, nil +} + +func (h *GitHandler) logger() *slog.Logger { + if h.proxy != nil && h.proxy.Logger != nil { + return h.proxy.Logger + } + return slog.Default() +} + +// gitSubmoduleLocation turns the URL .gitmodules gives a submodule into a +// location as Mirror takes it. A relative URL is resolved against the +// superproject's repoURL, as git does; ssh, git and scp-like URLs name the +// host and path the repository has there. +func gitSubmoduleLocation(repoURL, submoduleURL string) (string, error) { + var host, repoPath string + switch { + case strings.HasPrefix(submoduleURL, "./"), strings.HasPrefix(submoduleURL, "../"): + u, err := url.Parse(repoURL) + if err != nil { + return "", err + } + host, repoPath = u.Host, path.Join(u.Path, submoduleURL) + case strings.Contains(submoduleURL, "://"): + u, err := url.Parse(submoduleURL) + if err != nil { + return "", err + } + switch u.Scheme { + case "http", "https": + host = u.Host + case "ssh", "git", "git+ssh", "ssh+git": + host = u.Hostname() + default: + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + repoPath = u.Path + default: + // scp-like: [user@]host:path + hostPart, p, ok := strings.Cut(submoduleURL, ":") + if !ok || strings.Contains(hostPart, "/") { + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + _, host, _ = strings.Cut(hostPart, "@") + if host == "" { + host = hostPart + } + repoPath = p + } + repoPath = strings.Trim(path.Clean("/"+repoPath), "/") + if host == "" || repoPath == "" { + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + return strings.ToLower(host) + "/" + repoPath, nil +} + +// mirrorRepository resolves location to a repository of the upstream.git +// host whose URL it starts with, the longest such URL if several do. +func (h *GitHandler) mirrorRepository(location string) (gitRepository, error) { + var best, bestPrefix, rest string + for name, upstream := range h.upstreams { + prefix := gitLocationPrefix(upstream.URL) + path, ok := strings.CutPrefix(location, prefix) + if prefix == "" || !ok || path == "" { + continue + } + if len(prefix) > len(bestPrefix) || (len(prefix) == len(bestPrefix) && name < best) { + best, bestPrefix, rest = name, prefix, path + } + } + if best == "" { + return gitRepository{}, fmt.Errorf("no upstream.git host serves %s", location) + } + repo, status, msg := h.repository(best + "/" + rest) + if status != 0 { + return gitRepository{}, fmt.Errorf("%s: %s", location, msg) + } + return repo, nil +} + +// gitLocationPrefix is how a pkg:git PURL names the repositories under an +// upstream URL: its host and path, ending in a slash. Empty for a URL +// without a host. +func gitLocationPrefix(upstreamURL string) string { + u, err := url.Parse(upstreamURL) + if err != nil || u.Host == "" { + return "" + } + return strings.ToLower(u.Host) + strings.TrimSuffix(u.Path, "/") + "/" +} diff --git a/internal/handler/git_mirror_test.go b/internal/handler/git_mirror_test.go new file mode 100644 index 0000000..227bd71 --- /dev/null +++ b/internal/handler/git_mirror_test.go @@ -0,0 +1,154 @@ +package handler + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/config" +) + +func TestGitMirrorLocation(t *testing.T) { + h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ + "github": {URL: "https://github.com"}, + "wikimedia": {URL: "https://gitlab.wikimedia.org", Repos: []string{"mediawiki/**"}}, + "sub": {URL: "https://git.example/r/"}, + "nested": {URL: "https://git.example/r/deep"}, + }}) + tests := []struct { + location string + ok bool + key, repoURL string + }{ + {"github.com/acme/widgets", true, h.upstreams["github"].dir + "/acme/widgets.git", "https://github.com/acme/widgets"}, + {"github.com/acme/widgets.git", true, h.upstreams["github"].dir + "/acme/widgets.git", "https://github.com/acme/widgets.git"}, + {"gitlab.wikimedia.org/mediawiki/skins/Vector", true, h.upstreams["wikimedia"].dir + "/mediawiki/skins/Vector.git", "https://gitlab.wikimedia.org/mediawiki/skins/Vector"}, + {"gitlab.wikimedia.org/other/repo", false, "", ""}, + {"git.example/r/a", true, h.upstreams["sub"].dir + "/a.git", "https://git.example/r/a"}, + // The longest matching upstream URL wins. + {"git.example/r/deep/a", true, h.upstreams["nested"].dir + "/a.git", "https://git.example/r/deep/a"}, + {"git.example/other", false, "", ""}, + {"git.example/r", false, "", ""}, + {"github.com.evil/acme/widgets", false, "", ""}, + {"github.com/acme/../widgets", false, "", ""}, + {"codeberg.org/forgejo/forgejo", false, "", ""}, + } + for _, tt := range tests { + t.Run(tt.location, func(t *testing.T) { + repo, err := h.mirrorRepository(tt.location) + if (err == nil) != tt.ok { + t.Fatalf("error %v, want ok %v", err, tt.ok) + } + if tt.ok && (repo.key != tt.key || repo.url != tt.repoURL) { + t.Errorf("key %q, URL %q; want %q, %q", repo.key, repo.url, tt.key, tt.repoURL) + } + }) + } +} + +func TestGitMirrorCachesCommitForClones(t *testing.T) { + g := newGitTestSetup(t, 0, 0) + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + location := strings.TrimPrefix(g.upstreamURL, "http://") + "/repo" + + got, err := g.handler.Mirror(context.Background(), location, sha) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("first mirror = %+v, want fetched", got) + } + got, err = g.handler.Mirror(context.Background(), location, sha) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + + // With the upstream gone, a clone through the proxy gets the mirrored copy. + g.upstream.Close() + clone := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", g.url(), clone) + if head := runGitCommand(t, clone, "rev-parse", "HEAD"); head != sha { + t.Errorf("cloned HEAD %s, want %s", head, sha) + } +} + +func TestGitSubmoduleLocation(t *testing.T) { + tests := []struct { + repoURL, submoduleURL, want string + }{ + {"https://github.com/acme/widgets", "../lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets.git", "../../other/lib", "github.com/other/lib"}, + {"https://github.com/acme/widgets", "./vendored", "github.com/acme/widgets/vendored"}, + {"http://127.0.0.1:8080/team/repo", "../lib.git", "127.0.0.1:8080/team/lib.git"}, + {"https://github.com/acme/widgets", "https://GitLab.com/group/lib.git/", "gitlab.com/group/lib.git"}, + {"https://github.com/acme/widgets", "ssh://git@github.com:22/acme/lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets", "git://git.example/lib", "git.example/lib"}, + {"https://github.com/acme/widgets", "git@github.com:acme/lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets", "/srv/git/lib.git", ""}, + {"https://github.com/acme/widgets", "file:///srv/git/lib.git", ""}, + {"https://github.com/acme/widgets", "", ""}, + } + for _, tt := range tests { + got, err := gitSubmoduleLocation(tt.repoURL, tt.submoduleURL) + if tt.want == "" { + if err == nil { + t.Errorf("location of %q = %q, want an error", tt.submoduleURL, got) + } + continue + } + if err != nil || got != tt.want { + t.Errorf("location of %q from %s = %q, %v; want %q", tt.submoduleURL, tt.repoURL, got, err, tt.want) + } + } +} + +// TestGitMirrorCachesSubmodules mirrors a commit with a submodule served by +// the same upstream and one on a host no upstream.git serves, which is +// skipped. +func TestGitMirrorCachesSubmodules(t *testing.T) { + g := newGitTestSetup(t, 0, 0) + libBare := filepath.Join(g.root, "lib.git") + runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", libBare) + libWork := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", libBare, libWork) + runGitCommand(t, libWork, "commit", "--quiet", "--allow-empty", "-m", "lib") + runGitCommand(t, libWork, "push", "--quiet", "origin", "HEAD:main") + libCommit := runGitCommand(t, libWork, "rev-parse", "HEAD") + + gitmodules := "[submodule \"lib\"]\n\tpath = lib\n\turl = ../lib.git\n" + + "[submodule \"elsewhere\"]\n\tpath = elsewhere\n\turl = https://elsewhere.example/x.git\n" + if err := os.WriteFile(filepath.Join(g.work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { + t.Fatal(err) + } + runGitCommand(t, g.work, "add", ".gitmodules") + runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",lib") + runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+strings.Repeat("ab", 20)+",elsewhere") + runGitCommand(t, g.work, "commit", "--quiet", "-m", "submodules") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + location := strings.TrimPrefix(g.upstreamURL, "http://") + "/repo" + + got, err := g.handler.Mirror(context.Background(), location, sha) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("first mirror = %+v, want fetched", got) + } + if got, err = g.handler.Mirror(context.Background(), location, sha); err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + if !strings.Contains(g.logs.String(), "elsewhere.example") { + t.Errorf("log doesn't name the skipped submodule:\n%s", g.logs.String()) + } + + // With the upstream gone, the submodule's commit is served from the cache. + g.upstream.Close() + clone := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", g.server.URL+"/git/up/lib", clone) + if head := runGitCommand(t, clone, "rev-parse", "HEAD"); head != libCommit { + t.Errorf("cloned lib HEAD %s, want %s", head, libCommit) + } +} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go index 0aeff4b..aeec2bc 100644 --- a/internal/handler/git_test.go +++ b/internal/handler/git_test.go @@ -129,9 +129,16 @@ func runGitCommand(t *testing.T, dir string, args ...string) string { // gitTestSetup is a GitHandler in front of a cache of one upstream // repository, "repo", served by git http-backend. type gitTestSetup struct { - work string + work string + // root holds the upstream's bare repositories. + root string server *httptest.Server - logs *syncLog + // upstream serves the repository, at upstreamURL/repo, and handler is + // the GitHandler under test. + upstream *httptest.Server + upstreamURL string + handler *GitHandler + logs *syncLog // slow delays every upstream request by half a second. slow atomic.Bool // active counts the requests the handler is serving. @@ -156,7 +163,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe bare := filepath.Join(root, "repo.git") runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) runGitCommand(t, bare, "config", "uploadpack.allowAnySHA1InWant", "true") - g := &gitTestSetup{work: t.TempDir(), logs: &syncLog{}} + g := &gitTestSetup{work: t.TempDir(), root: root, logs: &syncLog{}} runGitCommand(t, "", "clone", "--quiet", bare, g.work) runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "first") runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") @@ -172,6 +179,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe backend.ServeHTTP(w, r) })) t.Cleanup(upstream.Close) + g.upstream, g.upstreamURL = upstream, upstream.URL logger := slog.New(slog.NewTextHandler(g.logs, nil)) cache, err := gitcache.New(gitcache.Options{ @@ -190,6 +198,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe if writeStall > 0 { h.writeStall = writeStall } + g.handler = h routes := http.StripPrefix("/git", h.Routes()) g.server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { g.active.Add(1) diff --git a/internal/handler/go.go b/internal/handler/go.go index 08dae57..14af7cf 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -23,7 +23,28 @@ const ( ) func init() { - retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: golangRetentionKey}) + retention.Default.Register(retention.Spec{ + Key: "golang", + CanonicalPackage: golangRetentionKey, + CompareVersions: golangCompareVersions, + }) +} + +// golangCompareVersions orders two module versions as the zips are cached, +// which is in the escaped form of the download URL (v1.0.0-!r!c1 for +// v1.0.0-RC1). +func golangCompareVersions(a, b string) int { + order := retention.VersionOrder("golang") + return order(unescapedGoVersion(a), unescapedGoVersion(b)) +} + +// unescapedGoVersion returns the version an escaped one stands for, or the +// input when it is not a valid escaped version. +func unescapedGoVersion(escaped string) string { + if version, err := gomodule.UnescapeVersion(escaped); err == nil { + return version + } + return escaped } func golangRetentionKey(p *purl.PURL) (string, bool) { diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index 8b38206..e96e5ab 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -383,7 +383,9 @@ func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, name, ups // the upstream timeout before the fetch falls back to it. if h.sumDBOffline && h.sumDBProbes.recentlyFailed(name) { if entry, body, contentType, ok := h.readCachedSumDB(r.Context(), cacheKey, check); ok { - p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, body, contentType, entry.ContentEncoding.String) + cm := p.cachedMetaFor(entry) + cm.stale = true // upstream was not asked + writeMetadataBody(w, r, cm, body, contentType, entry.ContentEncoding.String) return } } @@ -416,7 +418,7 @@ func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, name, ups return } - p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) + writeMetadataResult(w, r, res) } // fetchSumDB fetches a checksum database response through the metadata diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go index 6a0c47b..d47e432 100644 --- a/internal/handler/go_sumdb_test.go +++ b/internal/handler/go_sumdb_test.go @@ -985,3 +985,26 @@ func TestGoSumDB_ClientRoundTrip(t *testing.T) { t.Fatalf("lookup from the proxy cache with the database down: %v", err) } } + +// TestGoSumDB_OfflineCopyMarkedStaleWithoutTTL: with metadata_ttl 0 the +// cached copy offline mode serves while the database is unreachable still +// carries the Warning header. +func TestGoSumDB_OfflineCopyMarkedStaleWithoutTTL(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) + h := newSumDBTestHandler(t, db.server.URL, true, 0) + h.sumDBOffline = true + + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("online: status = %d, want 200", w.Code) + } + db.down.Store(true) + serveGoRequest(h, "/sumdb/sum.example"+lookup) + if !h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Fatal("the failure was not remembered") + } + w := serveGoRequest(h, "/sumdb/sum.example"+lookup) + if w.Code != http.StatusOK || w.Header().Get("Warning") != staleWarning { + t.Errorf("offline copy = %d Warning %q, want 200 %q", w.Code, w.Header().Get("Warning"), staleWarning) + } +} diff --git a/internal/handler/go_test.go b/internal/handler/go_test.go index ae998c9..514cefc 100644 --- a/internal/handler/go_test.go +++ b/internal/handler/go_test.go @@ -80,3 +80,27 @@ func TestLastComponent(t *testing.T) { } } } + +// Module zips are cached under the escaped version of the download URL, so +// the order for keep_latest unescapes before comparing. +func TestGolangCompareVersions(t *testing.T) { + tests := []struct { + a, b string + want int + }{ + {"v1.10.0", "v1.9.0", 1}, + {"v1.0.0", "v1.0.0-!r!c1", 1}, + {"v1.0.0-!r!c2", "v1.0.0-!r!c1", 1}, + {"v1.0.0-!r!c1", "v1.0.0-beta", -1}, + // Compared as stored, "!" would sort the escaped one below "0rc1". + {"v1.0.0-!r!c1", "v1.0.0-0rc1", 1}, + {"v2.0.0+incompatible", "v1.9.0", 1}, + {"v1.2.3", "v1.2.3", 0}, + } + for _, tt := range tests { + got := golangCompareVersions(tt.a, tt.b) + if (got > 0) != (tt.want > 0) || (got < 0) != (tt.want < 0) { + t.Errorf("golangCompareVersions(%q, %q) = %d, want sign of %d", tt.a, tt.b, got, tt.want) + } + } +} diff --git a/internal/handler/go_vulndb.go b/internal/handler/go_vulndb.go index d3c1598..4cd2970 100644 --- a/internal/handler/go_vulndb.go +++ b/internal/handler/go_vulndb.go @@ -57,7 +57,7 @@ func (h *GoHandler) handleVulnDB(w http.ResponseWriter, r *http.Request, path st p.writeMetadataFetchError(w, res.err) return } - p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) + writeMetadataResult(w, r, res) } // validGoVulnDBPath reports whether path is a file the vulnerability diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4b7ebe7..9786eea 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -4,7 +4,9 @@ package handler import ( "bytes" "context" + "crypto/sha256" "database/sql" + "encoding/hex" "errors" "fmt" "io" @@ -142,6 +144,10 @@ func ifNoneMatchHits(header, etag string) bool { // defaultMetadataMaxSize is used when Proxy.MetadataMaxSize is unset. const defaultMetadataMaxSize = 100 << 20 +// defaultMetadataStreamMaxSize is used when Proxy.MetadataStreamMaxSize is +// unset. +const defaultMetadataStreamMaxSize = 1 << 30 + // ErrMetadataTooLarge is returned when upstream metadata exceeds the configured limit. var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit") @@ -157,7 +163,9 @@ func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { // Reading into a buffer of that size avoids io.ReadAll growing its buffer // step by step, which peaks at more than twice the document size. size is // only a hint: a negative one, or one above the limit, reads as ReadMetadata -// does, and a body of a different length is still read correctly. +// does, and a body of a different length is still read correctly. A body +// read through upstreamBodyReader that upstream cuts off fails, even though +// it ends like a body shorter than size. func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { limit := p.MetadataMaxSize if limit <= 0 { @@ -170,7 +178,10 @@ func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { // the buffer and a longer one is noticed. buf := make([]byte, size+1) n, err := io.ReadFull(r, buf) + var cut *upstreamBodyError switch { + case errors.As(err, &cut): + return nil, err case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): if int64(n) < size/2 { // The hint was far off; don't keep the oversized buffer alive. @@ -215,24 +226,23 @@ type Proxy struct { Cooldown CooldownPolicy // ShortestCooldown is the shortest cooldown configured anywhere. It // bounds how long a rewrite filtered by cooldown is cached. - ShortestCooldown time.Duration - Denylist *denylist.Policy - CacheMetadata bool - MetadataTTL time.Duration - MetadataMaxSize int64 - GradleReadOnly bool - HostedEnabled bool - GradleMaxUploadSize int64 + ShortestCooldown time.Duration + Denylist *denylist.Policy + CacheMetadata bool + MetadataTTL time.Duration + MetadataMaxSize int64 + // MetadataStreamMaxSize caps a metadata document streamed from upstream + // into storage, which metadata_max_size doesn't cover. Default 1 GB. + MetadataStreamMaxSize int64 + GradleReadOnly bool + HostedEnabled bool + GradleMaxUploadSize int64 // NPMFullMetadata requests full npm packuments from upstream even when // cooldown is disabled, so served metadata carries publish times. NPMFullMetadata bool DirectServe bool DirectServeTTL time.Duration - // DirectServeBaseURL, if set, replaces the scheme and host of presigned - // URLs so clients receive a public address even when the proxy reaches - // storage at an internal one. - DirectServeBaseURL string - HTTPClient *http.Client + HTTPClient *http.Client // RateLimits is the gate under the upstream clients. Artifact fetches // check it before calling Fetcher, so held hosts never reach its breaker. @@ -276,10 +286,19 @@ type Proxy struct { metaMu sync.Mutex inFlightMeta map[string]*inflightMetadata + // metaDetachSlots caps the metadata fetches that keep running after + // their caller went away (see coalesceMetadata). Guarded by metaMu + // while it is created. + metaDetachSlots chan struct{} + // rewrites caches metadata documents after their handler rewrites them. // Nil leaves every request to rewrite its own copy. rewrites *rewriteCache + // publishIndexes keeps publish times read out of large metadata + // documents for the download cooldown check; see publishIndex. + publishIndexes publishIndexCache + // now is the clock rewritten metadata is dated by, and started the time // the process started; see metadataDate. now func() time.Time @@ -288,6 +307,11 @@ type Proxy struct { // cooldownWatch holds the rewrites recording which versions cooldown // holds back; see cooldown_expiry.go. cooldownWatch cooldownWatch + shared *Proxy + + // viewCooldown is set on the shared proxy once a view of it has cooldown + // on; see anyCooldownEnabled. + viewCooldown atomic.Bool } // cooldownEcosystems are the ecosystems whose handlers apply cooldown. Others @@ -439,6 +463,16 @@ func (p *Proxy) HasCachedArtifact(ctx context.Context, ecosystem, name, version, return p.Storage.Exists(ctx, artifact.StoragePath) } +// MarkMirrored records that a mirror run found an artifact cached, which +// keeps it from retention without counting as a cache hit. +func (p *Proxy) MarkMirrored(ecosystem, name, version, filename string) error { + _, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return err + } + return p.DB.MarkArtifactMirrored(versionPURL, filename) +} + // ClearCachedArtifact clears an artifact cache record after an external // integrity check fails, and queues its stored bytes for deletion. func (p *Proxy) ClearCachedArtifact(ecosystem, name, version, filename string) error { @@ -529,10 +563,16 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s storagePath: artifact.StoragePath, } + if servingHead(ctx) { + result.rangeable = readsSeekable(p.Storage) + p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) + return result, nil + } + if p.DirectServe { signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL) if err == nil { - result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL) + result.RedirectURL = signed p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) return result, nil } @@ -542,12 +582,6 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s } } - if servingHead(ctx) { - result.rangeable = readsSeekable(p.Storage) - p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) - return result, nil - } - start := time.Now() reader, err := p.Storage.Open(ctx, artifact.StoragePath) metrics.RecordStorageOperation("read", time.Since(start)) @@ -579,26 +613,6 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s return result, nil } -// rewriteSignedURLHost replaces the scheme and host of a signed URL with those -// from baseURL, preserving the path and query (which carry the signature). -// Returns signed unchanged if baseURL is empty or either URL fails to parse. -func rewriteSignedURLHost(signed, baseURL string) string { - if baseURL == "" { - return signed - } - s, err := url.Parse(signed) - if err != nil { - return signed - } - b, err := url.Parse(baseURL) - if err != nil || b.Scheme == "" || b.Host == "" { - return signed - } - s.Scheme = b.Scheme - s.Host = b.Host - return s.String() -} - func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) { _ = p.DB.RecordArtifactHit(versionPURL, filename) metrics.RecordArtifactCacheHit(ecosystem) @@ -770,6 +784,10 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil "path", storagePath, "error", qErr) return sharedArtifact, storagePath, errUnrecorded } + } else if mirrorFetch(ctx) { + if err := p.DB.MarkArtifactMirrored(versionPURL, filename); err != nil { + p.Logger.Warn("failed to record mirrored artifact", "error", err) + } } return sharedArtifact, storagePath, nil @@ -930,18 +948,19 @@ type inflightFetch struct { // proxy_cache_misses_total and upstream fetch observations is what coalescing // saved. func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { - p.fetchMu.Lock() - if p.inFlight == nil { - p.inFlight = make(map[string]*inflightFetch) + shared := p.sharedProxy() + shared.fetchMu.Lock() + if shared.inFlight == nil { + shared.inFlight = make(map[string]*inflightFetch) } - f, joined := p.inFlight[key] + f, joined := shared.inFlight[key] if joined { f.waiters++ } else { f = &inflightFetch{done: make(chan struct{})} - p.inFlight[key] = f + shared.inFlight[key] = f } - p.fetchMu.Unlock() + shared.fetchMu.Unlock() if !joined { if streamsCacheMiss(ctx) { @@ -969,28 +988,30 @@ func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (a // leaveFetch withdraws a waiter that gave up. Once the fetch has finished the // waiter was counted as holding any unrecorded object, so it releases it. func (p *Proxy) leaveFetch(f *inflightFetch) { - p.fetchMu.Lock() + shared := p.sharedProxy() + shared.fetchMu.Lock() select { case <-f.done: - p.fetchMu.Unlock() + shared.fetchMu.Unlock() if f.unrecorded != nil { f.unrecorded.release() } default: f.waiters-- - p.fetchMu.Unlock() + shared.fetchMu.Unlock() } } // runSharedFetch performs the fetch that joined callers are waiting on. It is // never abandoned early, and always releases the key and wakes the waiters. func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { + shared := p.sharedProxy() // Set before running so a panicking commit leaves waiters with an error // rather than a zero-valued artifact. f.err = errSharedFetchAbandoned defer func() { - p.fetchMu.Lock() - delete(p.inFlight, key) + shared.fetchMu.Lock() + delete(shared.inFlight, key) if f.unrecorded != nil { // Hand each waiter a hold, then drop the one kept while the // waiters were still unknown. @@ -998,7 +1019,7 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch defer f.unrecorded.release() } close(f.done) - p.fetchMu.Unlock() + shared.fetchMu.Unlock() }() // A caller checks the cache before reaching here, so a fetch that finished @@ -1119,6 +1140,20 @@ func readsSeekable(store storage.Storage) bool { return ok && s.ReadsSeekable() } +type mirrorFetchKey struct{} + +// MirrorFetch marks ctx as a mirror run's rather than a client's. An +// artifact fetched under it is recorded as mirrored, so it doesn't count as +// downloaded by a client. +func MirrorFetch(ctx context.Context) context.Context { + return context.WithValue(ctx, mirrorFetchKey{}, true) +} + +func mirrorFetch(ctx context.Context) bool { + mirror, _ := ctx.Value(mirrorFetchKey{}).(bool) + return mirror +} + type detachUpstreamFetchKey struct{} // DetachUpstreamFetch marks ctx so that an artifact fetch started under it @@ -1146,25 +1181,49 @@ func upstreamFetchDetached(ctx context.Context) bool { const maxDetachedFetches = 64 func (p *Proxy) acquireDetachSlot() bool { - p.fetchMu.Lock() - if p.detachSlots == nil { - p.detachSlots = make(chan struct{}, maxDetachedFetches) + shared := p.sharedProxy() + return trySlot(&shared.fetchMu, &shared.detachSlots, maxDetachedFetches) +} + +func (p *Proxy) releaseDetachSlot() { + shared := p.sharedProxy() + <-shared.detachSlots +} + +// maxDetachedMetadataFetches caps the metadata fetches that may outlive their +// caller, for the same reason as maxDetachedFetches: a client could request +// any number of uncached documents and hang up, leaving each fetch to run to +// the HTTP client's timeout. Fetches beyond the cap end with their caller. +const maxDetachedMetadataFetches = 64 + +func (p *Proxy) acquireMetadataDetachSlot() bool { + shared := p.sharedProxy() + return trySlot(&shared.metaMu, &shared.metaDetachSlots, maxDetachedMetadataFetches) +} + +func (p *Proxy) releaseMetadataDetachSlot() { + shared := p.sharedProxy() + <-shared.metaDetachSlots +} + +// trySlot takes one of size slots without waiting, creating the slots under +// mu on first use. It reports whether a slot was free. +func trySlot(mu *sync.Mutex, slots *chan struct{}, size int) bool { + mu.Lock() + if *slots == nil { + *slots = make(chan struct{}, size) } - slots := p.detachSlots - p.fetchMu.Unlock() + ch := *slots + mu.Unlock() select { - case slots <- struct{}{}: + case ch <- struct{}{}: return true default: return false } } -func (p *Proxy) releaseDetachSlot() { - <-p.detachSlots -} - // fetchedArtifact is what a shared fetch hands its callers: metadata and a // storage path, neither holding reader state. type fetchedArtifact struct { @@ -1445,6 +1504,21 @@ func copyArtifactBody(w http.ResponseWriter, reader io.Reader, expectedSize int6 } } +// upstreamRequest sends r's method to upstreamURL, copying only the +// forwardHeaders that r sets. +func (p *Proxy) upstreamRequest(r *http.Request, upstreamURL string, forwardHeaders []string) (*http.Response, error) { + req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) + if err != nil { + return nil, err + } + for _, header := range forwardHeaders { + if v := r.Header.Get(header); v != "" { + req.Header.Set(header, v) + } + } + return p.HTTPClient.Do(req) +} + // ProxyUpstream forwards a request to an upstream URL without caching. // It copies the request, forwards specified headers, and streams the response back. // forwardHeaders controls the request headers sent upstream. End-to-end response @@ -1452,20 +1526,13 @@ func copyArtifactBody(w http.ResponseWriter, reader io.Reader, expectedSize int6 func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamURL string, forwardHeaders []string) { p.Logger.Debug("proxying to upstream", "url", upstreamURL) - req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) - if err != nil { - http.Error(w, "failed to create request", http.StatusInternalServerError) - return - } + resp, err := p.upstreamRequest(r, upstreamURL, forwardHeaders) + p.relayUpstream(w, r, resp, err) +} - // Copy request headers that affect content negotiation / caching - for _, header := range forwardHeaders { - if v := r.Header.Get(header); v != "" { - req.Header.Set(header, v) - } - } - - resp, err := p.HTTPClient.Do(req) +// relayUpstream writes the result of an upstream request: the response, or +// a 502 when the request failed. +func (p *Proxy) relayUpstream(w http.ResponseWriter, r *http.Request, resp *http.Response, err error) { if err != nil { if writeUpstreamHeld(w, err) { return @@ -1586,8 +1653,7 @@ var errStale304 = fmt.Errorf("upstream returned 304 but cached file is missing") // cacheKey is typically the package name but can include subpath components. // Optional acceptHeaders specify the Accept header(s) to send; defaults to application/json. func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL string, acceptHeaders ...string) ([]byte, string, error) { - body, contentType, _, err := p.fetchOrCacheMetadata(ctx, ecosystem, cacheKey, upstreamURL, "", nil, acceptHeaders...) - return body, contentType, err + return p.fetchOrCacheMetadata(ctx, ecosystem, cacheKey, upstreamURL, "", nil, acceptHeaders...) } // fetchOrCacheMetadata implements FetchOrCacheMetadata. acceptEncoding controls @@ -1603,13 +1669,13 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u // it can also decode the document into request-local state; a caller that // joined a fetch gets its own validation error directly. It must not modify the // body, which joined callers share. -func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, acceptEncoding string, validate func([]byte) error, acceptHeaders ...string) ([]byte, string, string, error) { +func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, acceptEncoding string, validate func([]byte) error, acceptHeaders ...string) ([]byte, string, error) { accept := contentTypeJSON if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { accept = acceptHeaders[0] } res := p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) - return res.body, res.contentType, res.contentEncoding, res.err + return res.body, res.contentType, res.err } // fetchMetadataDocument is FetchOrCacheMetadata for callers that rewrite the @@ -1749,23 +1815,29 @@ func (p *Proxy) refreshMetadata(ctx context.Context, ecosystem, cacheKey, upstre // and falls back to the cached copy if upstream fails. entry is the row to // revalidate against, or nil. func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error, entry *database.MetadataCacheEntry) metadataResult { - meta, err := p.fetchUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) - if errors.Is(err, errStale304) { - // 304 but cached file is gone; retry without ETag - meta, err = p.fetchUpstreamMetadata(ctx, upstreamURL, nil, accept, acceptEncoding) + fetch := func(entry *database.MetadataCacheEntry) (*upstreamMetadata, error) { + meta, err := p.fetchUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) + if err == nil && validate != nil { + err = validate(meta.body) + } + return meta, err } - if err == nil && validate != nil { - err = validate(meta.body) + meta, err := fetch(entry) + var digest string + // Validated first, so a 304 for a rejected copy doesn't mark it fresh. + if err == nil && p.CacheMetadata { + digest, err = p.refreshMetadataBlob(ecosystem, cacheKey, meta) + } + if errors.Is(err, errStale304) { + // 304 but the cached copy is gone or was replaced; retry without + // validators. + meta, err = fetch(nil) } if err == nil { - var digest string - if p.CacheMetadata { - digest = p.refreshMetadataBlob(ecosystem, cacheKey, meta) - if digest == "" { - digest = p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) - } + if p.CacheMetadata && digest == "" { + digest = p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) } - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} + return fetchedResult(meta, digest) } return p.staleMetadata(ctx, ecosystem, cacheKey, validate, entry, err) @@ -1787,17 +1859,25 @@ func (p *Proxy) staleMetadata(ctx context.Context, ecosystem, cacheKey string, v // Re-read the row so the encoding describes the blob as it is now: a // concurrent refetch may have replaced both since entry was read above - // (an identity blob swapped for a gzip one during rollout). - entry = p.currentMetadataEntry(ecosystem, cacheKey, entry) - - data, ct, readErr := p.readCachedMetadata(ctx, entry, validate) - if readErr != nil { - return metadataResult{err: fmt.Errorf("upstream failed and cached metadata unusable (%v): %w", readErr, err)} + // (an identity blob swapped for a gzip one during rollout). Bytes that + // don't match the row were replaced after it was read, so the row is + // read once more; if they still don't match, no row describes them. + for attempt := 0; ; attempt++ { + entry = p.currentMetadataEntry(ecosystem, cacheKey, entry) + data, ct, readErr := p.readCachedMetadata(ctx, entry, validate) + if readErr == nil && !matchesEntryDigest(entry, data) { + if attempt == 0 { + continue + } + readErr = errors.New("replaced while being read") + } + if readErr != nil { + return metadataResult{err: fmt.Errorf("upstream failed and cached metadata unusable (%v): %w", readErr, err)} + } + p.Logger.Info("serving metadata from cache", + "ecosystem", ecosystem, "key", cacheKey) + return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true, upstreamErr: err}.describedBy(entry) } - - p.Logger.Info("serving metadata from cache", - "ecosystem", ecosystem, "key", cacheKey) - return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true}.describedBy(entry) } // metadataResult is what a metadata lookup hands back. A shared fetch gives @@ -1810,23 +1890,54 @@ type metadataResult struct { // empty otherwise. digest string err error + // etag and lastModified are the validators of body: upstream's for a + // fetched document, the cache row's for a stored one. + etag string + lastModified time.Time // stale is set when the body is the cached copy served because upstream - // failed. - stale bool + // failed, and upstreamErr is that failure. + stale bool + upstreamErr error // fromCache is set when the body is a stored copy within its TTL, read // without asking upstream. fromCache bool + // entry is the cache row of the stored document, for a lookup that + // leaves the document in storage (see storedMetadataDocument). body is + // nil then. + entry *database.MetadataCacheEntry } -// describedBy returns res with the digest that entry, the cache row its body -// was read for, records. +// describedBy returns res with the digest and validators that entry, the +// cache row its body was read for, records. func (res metadataResult) describedBy(entry *database.MetadataCacheEntry) metadataResult { if entry.ContentDigest.Valid { res.digest = entry.ContentDigest.String } + res.etag = entry.ETag.String + if entry.LastModified.Valid { + res.lastModified = entry.LastModified.Time + } return res } +// fetchedResult returns the result for meta, a document from upstream (or +// the stored copy it answered 304 for), cached under digest. +func fetchedResult(meta *upstreamMetadata, digest string) metadataResult { + return metadataResult{ + body: meta.body, + contentType: meta.contentType, + contentEncoding: meta.contentEncoding, + digest: digest, + etag: meta.etag, + lastModified: meta.lastModified, + } +} + +// cachedMeta returns the validators and staleness of res's body. +func (res metadataResult) cachedMeta() cachedMeta { + return cachedMeta{etag: res.etag, lastModified: res.lastModified, stale: res.stale} +} + // inflightMetadata is one metadata fetch that concurrent callers share. res is // written before done closes and read only after, so the close is the handoff. type inflightMetadata struct { @@ -1855,20 +1966,23 @@ func metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncodin // metadata fetch has no scan or mirror that depends on the caller aborting it, // and CI jobs asking for the same Composer or npm metadata would otherwise all // fail when the first of them disconnects. It stays bounded by the HTTP -// client's timeout. Waiters still leave when their own context ends. +// client's timeout, and at most maxDetachedMetadataFetches run detached at +// once; beyond that, fetch ends with the first caller. Waiters still leave +// when their own context ends. func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(context.Context) metadataResult) (metadataResult, bool) { - p.metaMu.Lock() - if p.inFlightMeta == nil { - p.inFlightMeta = make(map[string]*inflightMetadata) + shared := p.sharedProxy() + shared.metaMu.Lock() + if shared.inFlightMeta == nil { + shared.inFlightMeta = make(map[string]*inflightMetadata) } - f, joined := p.inFlightMeta[key] + f, joined := shared.inFlightMeta[key] if joined { f.waiters++ } else { f = &inflightMetadata{done: make(chan struct{})} - p.inFlightMeta[key] = f + shared.inFlightMeta[key] = f } - p.metaMu.Unlock() + shared.metaMu.Unlock() if joined { select { @@ -1883,12 +1997,17 @@ func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(con // rather than an empty body. f.res = metadataResult{err: errSharedFetchAbandoned} defer func() { - p.metaMu.Lock() - delete(p.inFlightMeta, key) - p.metaMu.Unlock() + shared.metaMu.Lock() + delete(shared.inFlightMeta, key) + shared.metaMu.Unlock() close(f.done) }() - f.res = fetch(context.WithoutCancel(ctx)) + fetchCtx := ctx + if p.acquireMetadataDetachSlot() { + defer p.releaseMetadataDetachSlot() + fetchCtx = context.WithoutCancel(ctx) + } + f.res = fetch(fetchCtx) return f.res, false } @@ -1943,6 +2062,8 @@ type upstreamMetadata struct { contentEncoding string etag string lastModified time.Time + // url is the upstream URL the document was fetched from. + url string // notModified is the cache row upstream answered 304 for, whose stored // bytes body holds. It is nil for a document upstream sent. notModified *database.MetadataCacheEntry @@ -1955,6 +2076,67 @@ type upstreamMetadata struct { // alongside for the caller to store and replay. An empty acceptEncoding leaves Go to negotiate and // decompress transparently. func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept, acceptEncoding string) (*upstreamMetadata, error) { + resp, err := p.requestUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + // 304 Not Modified -- our cached copy is still good + if resp.StatusCode == http.StatusNotModified && entry != nil { + cached, readErr := p.Storage.Open(ctx, entry.StoragePath) + if readErr != nil { + return nil, errStale304 + } + defer func() { _ = cached.Close() }() + data, readErr := p.ReadMetadataSized(cached, metadataEntrySize(entry)) + if readErr != nil || !matchesEntryDigest(entry, data) { + // Gone, or replaced by another instance since entry was read. + return nil, errStale304 + } + meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, url: upstreamURL, notModified: entry} + if entry.ContentType.Valid { + meta.contentType = entry.ContentType.String + } + if entry.ContentEncoding.Valid { + meta.contentEncoding = entry.ContentEncoding.String + } + if entry.LastModified.Valid { + meta.lastModified = entry.LastModified.Time + } + return meta, nil + } + + if err := upstreamMetadataStatus(resp); err != nil { + return nil, err + } + + // ContentLength is -1 when the transport decompresses the body, and + // otherwise the length of the bytes read here. + body, err := p.ReadMetadataSized(upstreamBodyReader{resp.Body}, resp.ContentLength) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + + meta := &upstreamMetadata{ + body: body, + contentType: resp.Header.Get(headerContentType), + contentEncoding: resp.Header.Get(headerContentEncoding), + etag: resp.Header.Get(headerETag), + url: upstreamURL, + } + if meta.contentType == "" { + meta.contentType = contentTypeJSON + } + if lm := resp.Header.Get(headerLastModified); lm != "" { + meta.lastModified, _ = http.ParseTime(lm) + } + return meta, nil +} + +// requestUpstreamMetadata sends the request for a metadata document, +// revalidating entry when it is given. +func (p *Proxy) requestUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept, acceptEncoding string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) if err != nil { return nil, fmt.Errorf("creating request: %w", err) @@ -1974,97 +2156,86 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e if err != nil { return nil, fmt.Errorf("fetching metadata: %w", err) } - defer func() { _ = resp.Body.Close() }() + return resp, nil +} - // 304 Not Modified -- our cached copy is still good - if resp.StatusCode == http.StatusNotModified && entry != nil { - cached, readErr := p.Storage.Open(ctx, entry.StoragePath) - if readErr != nil { - return nil, errStale304 - } - defer func() { _ = cached.Close() }() - data, readErr := p.ReadMetadataSized(cached, metadataEntrySize(entry)) - if readErr != nil { - return nil, errStale304 - } - meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, notModified: entry} - if entry.ContentType.Valid { - meta.contentType = entry.ContentType.String - } - if entry.ContentEncoding.Valid { - meta.contentEncoding = entry.ContentEncoding.String - } - if entry.LastModified.Valid { - meta.lastModified = entry.LastModified.Time - } - return meta, nil +// matchesEntryDigest reports whether data is the document entry records, +// when entry has a sha256 digest to check it by. +func matchesEntryDigest(entry *database.MetadataCacheEntry, data []byte) bool { + want, ok := strings.CutPrefix(entry.ContentDigest.String, "sha256:") + if !entry.ContentDigest.Valid || !ok { + return true } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) == want +} - if resp.StatusCode == http.StatusNotFound { +// upstreamMetadataStatus returns the error for a metadata response other +// than 200. Callers handle a 304 they asked for before. +func upstreamMetadataStatus(resp *http.Response) error { + switch resp.StatusCode { + case http.StatusOK: + return nil + case http.StatusNotFound: body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamNotFoundBodyLimit)) - return nil, &upstreamNotFoundError{body: body} + return &upstreamNotFoundError{body: body} } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("upstream returned %d", resp.StatusCode) - } - - // ContentLength is -1 when the transport decompresses the body, and - // otherwise the length of the bytes read here. - body, err := p.ReadMetadataSized(resp.Body, resp.ContentLength) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - - meta := &upstreamMetadata{ - body: body, - contentType: resp.Header.Get(headerContentType), - contentEncoding: resp.Header.Get(headerContentEncoding), - etag: resp.Header.Get(headerETag), - } - if meta.contentType == "" { - meta.contentType = contentTypeJSON - } - if lm := resp.Header.Get(headerLastModified); lm != "" { - meta.lastModified, _ = http.ParseTime(lm) - } - return meta, nil + return fmt.Errorf("upstream returned %d", resp.StatusCode) } // refreshMetadataBlob records that upstream answered 304 for the cached // document meta was read from, by moving the row's fetched_at on. The bytes in // storage are unchanged, so they are neither written nor hashed again. It // returns the row's digest, or "" when the caller has to store meta as for a -// new document: when upstream sent one, the row records no digest, or a -// concurrent fetch has replaced the document since it was read. -func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMetadata) string { +// new document: when upstream sent one or the row records no digest. When +// a concurrent fetch has replaced the document since it was read, it returns +// errStale304: storing meta would put the old document back under the new +// one. A failed database write only leaves fetched_at behind, so the +// confirmed bytes are still served. +func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMetadata) (string, error) { entry := meta.notModified if entry == nil || p.DB == nil || !entry.ContentDigest.Valid { - return "" + return "", nil } refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) if err != nil { + // The bytes were checked against the row and upstream confirmed + // them; only fetched_at stays behind, so they are still served. p.Logger.Warn("failed to refresh cached metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return "" + return entry.ContentDigest.String, nil } if !refreshed { - return "" + return "", errStale304 + } + if !entry.UpstreamURL.Valid && meta.url != "" { + if err := p.DB.SetMetadataUpstreamURL(ecosystem, cacheKey, meta.url); err != nil { + p.Logger.Warn("failed to record metadata upstream URL", "ecosystem", ecosystem, "key", cacheKey, "error", err) + } } p.recordMetadataRead(entry) - return entry.ContentDigest.String + return entry.ContentDigest.String, nil } // cacheMetadataBlob stores metadata bytes in storage and updates the // database. It returns the digest it recorded for the bytes, or "" when they // were not cached. func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, meta *upstreamMetadata) string { + digest, _ := p.storeMetadataBlob(ctx, ecosystem, cacheKey, storagePath, bytes.NewReader(meta.body), meta) + return digest +} + +// storeMetadataBlob is cacheMetadataBlob for a document read from body, +// which meta describes. It also returns the error that kept it from caching +// the document. +func (p *Proxy) storeMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, body io.Reader, meta *upstreamMetadata) (string, error) { if p.DB == nil || p.Storage == nil { - return "" + return "", errMetadataNotStored } - size, hash, err := p.Storage.Store(ctx, storagePath, bytes.NewReader(meta.body)) + size, hash, err := p.Storage.Store(ctx, storagePath, body) if err != nil { p.Logger.Warn("failed to cache metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return "" + return "", err } var digest string if hash != "" { @@ -2081,6 +2252,7 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor // The digest identifies the stored bytes, so a rewrite cached for them // can be found without reading them back (see storedRewrite). ContentDigest: sql.NullString{String: digest, Valid: digest != ""}, + UpstreamURL: sql.NullString{String: meta.url, Valid: meta.url != ""}, Size: sql.NullInt64{Int64: size, Valid: true}, LastModified: sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, @@ -2094,9 +2266,9 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { p.Logger.Warn("failed to discard metadata blob", "ecosystem", ecosystem, "key", cacheKey, "error", delErr) } - return "" + return "", err } - return digest + return digest, nil } // currentMetadataEntry re-reads the metadata cache row and returns it, or @@ -2117,18 +2289,6 @@ type cachedMeta struct { stale bool } -// lookupCachedMeta retrieves cache validators for a metadata entry. -func (p *Proxy) lookupCachedMeta(ecosystem, cacheKey string) cachedMeta { - if p.DB == nil { - return cachedMeta{} - } - entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) - if err != nil || entry == nil { - return cachedMeta{} - } - return p.cachedMetaFor(entry) -} - // cachedMetaFor returns the cache validators and freshness state of entry. func (p *Proxy) cachedMetaFor(entry *database.MetadataCacheEntry) cachedMeta { var cm cachedMeta @@ -2151,7 +2311,15 @@ func (p *Proxy) cachedMetaFor(entry *database.MetadataCacheEntry) cachedMeta { // When metadata caching is disabled, the response is streamed directly to avoid buffering // large metadata responses (e.g. npm packages with many versions) in memory. func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey string, acceptHeaders ...string) { - p.proxyCachedWithEncoding(w, r, upstreamURL, ecosystem, cacheKey, "identity", acceptHeaders...) + p.proxyCachedWithEncoding(w, r, upstreamURL, ecosystem, cacheKey, "identity", nil, acceptHeaders...) +} + +// metadataHooks are optional callbacks for proxyCachedWithEncoding: hit runs +// after a fresh cached row is served, with that row and when it was read, and +// fetched runs with a fetched document before it is written. +type metadataHooks struct { + hit func(entry *database.MetadataCacheEntry, seen time.Time) + fetched func(body []byte, contentType, contentEncoding string) } // proxyCachedWithEncoding is ProxyCached with an explicit upstream Accept-Encoding. @@ -2159,24 +2327,39 @@ func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, // compressed for large, non-hash-pinned metadata whose clients decode gzip // (Homebrew API and Conda repodata). The stored bytes and Content-Encoding are // replayed verbatim either way. -func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, acceptEncoding string, acceptHeaders ...string) { +func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, acceptEncoding string, hooks *metadataHooks, acceptHeaders ...string) { if !p.CacheMetadata { // Stream directly without buffering when caching is off. p.proxyMetadataStream(w, r, upstreamURL, acceptEncoding, acceptHeaders...) return } - if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { + if hooks != nil && hooks.hit != nil { + if entry := p.freshMetadataEntry(ecosystem, cacheKey); entry != nil { + seen := time.Now() + if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { + hooks.hit(entry, seen) + return + } + } + } else if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { return } - body, contentType, contentEncoding, err := p.fetchOrCacheMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, acceptEncoding, nil, acceptHeaders...) - if err != nil { - p.writeMetadataFetchError(w, err) + accept := contentTypeJSON + if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { + accept = acceptHeaders[0] + } + res := p.lookupMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, nil) + if res.err != nil { + p.writeMetadataFetchError(w, res.err) return } - p.writeMetadataCachedResponseWithEncoding(w, r, ecosystem, cacheKey, body, contentType, contentEncoding) + if hooks != nil && hooks.fetched != nil { + hooks.fetched(res.body, res.contentType, res.contentEncoding) + } + writeMetadataResult(w, r, res) } // writeMetadataFetchError answers a failed metadata fetch. @@ -2192,19 +2375,11 @@ func (p *Proxy) writeMetadataFetchError(w http.ResponseWriter, err error) { http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) } -// writeMetadataCachedResponse writes a cached metadata response and handles -// conditional request headers using metadata cache validators. -func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType string) { - p.writeMetadataCachedResponseWithEncoding(w, r, ecosystem, cacheKey, body, contentType, "") -} - -// writeMetadataCachedResponseWithEncoding is writeMetadataCachedResponse with -// an explicit Content-Encoding. contentEncoding must describe the body being -// written; it is passed in rather than re-read from the cache row, which is -// missing or stale when the metadata cache write failed and would otherwise -// mislabel the bytes. -func (p *Proxy) writeMetadataCachedResponseWithEncoding(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType, contentEncoding string) { - writeMetadataBody(w, r, p.lookupCachedMeta(ecosystem, cacheKey), body, contentType, contentEncoding) +// writeMetadataResult writes the document res holds with its own validators +// and staleness, rather than the cache row's, which may describe other bytes +// when the cache write failed or another instance replaced it. +func writeMetadataResult(w http.ResponseWriter, r *http.Request, res metadataResult) { + writeMetadataBody(w, r, res.cachedMeta(), res.body, res.contentType, res.contentEncoding) } // writeMetadataBody writes a metadata document with the validators and @@ -2304,6 +2479,23 @@ func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", "") } +// cacheFilenameKeepingLegacy returns legacy when an artifact recorded under +// it was fetched from downloadURL, and filename otherwise. +func (p *Proxy) cacheFilenameKeepingLegacy(ecosystem, name, version, filename, legacy, downloadURL string) (string, error) { + _, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return "", err + } + record, err := p.DB.GetArtifact(versionPURL, legacy) + if err != nil { + return "", fmt.Errorf("checking artifact cache: %w", err) + } + if record != nil && record.UpstreamURL == downloadURL { + return legacy, nil + } + return filename, nil +} + // GetOrFetchArtifactFromURLWithHeaders retrieves an artifact from cache or fetches from a URL // with additional request-specific HTTP headers. func (p *Proxy) GetOrFetchArtifactFromURLWithHeaders(ctx context.Context, ecosystem, name, version, filename, downloadURL string, headers http.Header) (*CacheResult, error) { @@ -2482,6 +2674,39 @@ func (p *Proxy) getCachedArtifactWithUpstreamHash(ctx context.Context, pkgPURL, return nil, nil } +// refetchArtifactFromURL fetches a file upstream replaces in place, which +// a cached copy can't stand in for while upstream answers. The new copy +// replaces the cached one only once it is stored, and the cached one is +// served when the fetch fails for any reason but the file being missing or +// blocked. Concurrent callers share one fetch. +func (p *Proxy) refetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string) (*CacheResult, error) { + pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return nil, err + } + // Streaming never serves a cached copy, and a denied version is refused + // there. + if p.StreamArtifacts || p.versionDenied(ecosystem, name, version) || p.Denylist.Denied(versionPURL) { + return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", "") + } + metrics.RecordArtifactCacheMiss(ecosystem) + key := artifactCoalesceKey(versionPURL, filename, downloadURL, "", "") + noRecheck := func() (artifacts.Artifact, string, bool) { return artifacts.Artifact{}, "", false } + result, err := p.coalesceFetch(ctx, key, noRecheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { + return p.fetchAndCacheFromURL(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, nil, "", "") + }) + if err == nil || errors.Is(err, ErrUpstreamNotFound) || errors.Is(err, ErrArtifactBlocked) || ctx.Err() != nil { + return result, err + } + cached, cacheErr := p.checkCache(ctx, pkgPURL, versionPURL, filename) + if cacheErr != nil || cached == nil { + return nil, err + } + p.Logger.Warn("upstream fetch failed, serving cached copy", + "purl", versionPURL, "filename", filename, "error", err) + return cached, nil +} + // discardStaleArtifact clears the cached entry when its digest disagrees with // upstreamHash, queueing its object for deletion. It runs under the coalescing // key, after the recheck, so an entry a previous fetch refreshed is kept. diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 8ce636c..8c5e798 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -13,6 +13,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -638,74 +639,6 @@ func TestGetOrFetchArtifact_DirectServe_Redirect(t *testing.T) { } } -func TestGetOrFetchArtifact_DirectServe_BaseURLRewrite(t *testing.T) { - proxy, db, store, _ := setupTestProxy(t) - seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") - - proxy.DirectServe = true - proxy.DirectServeBaseURL = "https://cdn.example.com" - store.signedURL = "http://127.0.0.1:9000/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900" - - result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - want := "https://cdn.example.com/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900" - if result.RedirectURL != want { - t.Errorf("RedirectURL = %q, want %q", result.RedirectURL, want) - } -} - -func TestRewriteSignedURLHost(t *testing.T) { - tests := []struct { - name string - signed string - baseURL string - want string - }{ - { - "empty base url is no-op", - "http://127.0.0.1:9000/bucket/key?sig=abc", - "", - "http://127.0.0.1:9000/bucket/key?sig=abc", - }, - { - "replaces scheme and host", - "http://127.0.0.1:9000/bucket/key?sig=abc", - "https://cdn.example.com", - "https://cdn.example.com/bucket/key?sig=abc", - }, - { - "preserves path and query", - "http://minio:9000/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101", - "https://files.example.com", - "https://files.example.com/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101", - }, - { - "ignores base url path", - "http://127.0.0.1:9000/bucket/key?sig=abc", - "https://cdn.example.com/ignored", - "https://cdn.example.com/bucket/key?sig=abc", - }, - { - "invalid base url is no-op", - "http://127.0.0.1:9000/bucket/key?sig=abc", - "://bad", - "http://127.0.0.1:9000/bucket/key?sig=abc", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := rewriteSignedURLHost(tt.signed, tt.baseURL) - if got != tt.want { - t.Errorf("rewriteSignedURLHost(%q, %q) = %q, want %q", tt.signed, tt.baseURL, got, tt.want) - } - }) - } -} - func TestGetOrFetchArtifact_DirectServe_FallbackOnUnsupported(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") @@ -1584,3 +1517,54 @@ func TestCanonicalPackagePURLMatchesConfig(t *testing.T) { }) } } + +func TestProxyCachedRecordsUpstreamURL(t *testing.T) { + proxy, upstream := setupCachedProxy(t, `"abc123"`, "") + req := httptest.NewRequest(http.MethodGet, "/test", nil) + proxy.ProxyCached(httptest.NewRecorder(), req, upstream.URL+"/test", "test-eco", "url-key") + + entry, err := proxy.DB.GetMetadataCache("test-eco", "url-key") + if err != nil || entry == nil { + t.Fatalf("GetMetadataCache: %+v, %v", entry, err) + } + if entry.UpstreamURL.String != upstream.URL+"/test" { + t.Errorf("upstream_url = %q, want %q", entry.UpstreamURL.String, upstream.URL+"/test") + } +} + +// A row cached before upstream URLs were recorded gets its URL on the next +// revalidation, even when upstream answers 304. +func TestProxyCachedRecordsUpstreamURLOnNotModified(t *testing.T) { + var notModified atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"v1"`) + if r.Header.Get("If-None-Match") == `"v1"` { + notModified.Add(1) + w.WriteHeader(http.StatusNotModified) + return + } + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.HTTPClient = upstream.Client() + + proxy.ProxyCached(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/test", nil), upstream.URL+"/test", "test-eco", "url-304-key") + if _, err := proxy.DB.Exec("UPDATE metadata_cache SET upstream_url = NULL"); err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + proxy.ProxyCached(w, httptest.NewRequest(http.MethodGet, "/test", nil), upstream.URL+"/test", "test-eco", "url-304-key") + if w.Code != http.StatusOK || notModified.Load() != 1 { + t.Fatalf("status = %d, upstream 304s = %d; want 200 after one 304", w.Code, notModified.Load()) + } + entry, err := proxy.DB.GetMetadataCache("test-eco", "url-304-key") + if err != nil || entry == nil { + t.Fatalf("GetMetadataCache: %+v, %v", entry, err) + } + if entry.UpstreamURL.String != upstream.URL+"/test" { + t.Errorf("upstream_url after a 304 = %q, want %q", entry.UpstreamURL.String, upstream.URL+"/test") + } +} diff --git a/internal/handler/helm.go b/internal/handler/helm.go index fdc000e..661ca98 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -120,6 +120,24 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } + var charts *helmChartLookup + if h.proxy.cooldownEnabled() { + var err error + charts, err = h.downloadChartLookup(r, repository, upstreamURL) + if err != nil { + cached, cacheErr := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename) + if cacheErr == nil && cached != nil { + h.serveChart(w, r, repository, digest, filename, cached) + } else { + h.proxy.writeCooldownLookupError(w, helmMetadataEcosystem, repository, err) + } + return + } + if h.withholdChart(w, charts, digest, filename) { + return + } + } + cached, err := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename) if err != nil { h.proxy.Logger.Error("failed to check Helm chart cache", "error", err) @@ -131,13 +149,17 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } - index := h.fetchIndex(r.Context(), repository, upstreamURL) - if index.err != nil { - h.serveIndexError(w, index.err) - return + var downloadURL string + if charts == nil { + index := h.fetchIndex(r.Context(), repository, upstreamURL) + if index.err != nil { + h.serveIndexError(w, index.err) + return + } + downloadURL, err = h.findChartDownload(repository, upstreamURL, index, digest, filename) + } else { + downloadURL, err = h.findChartURL(charts, digest, filename) } - - downloadURL, err := h.findChartDownload(repository, upstreamURL, index, digest, filename) if err != nil { if errors.Is(err, errHelmChartNotFound) { http.NotFound(w, r) @@ -265,6 +287,10 @@ func (h *HelmHandler) findChartDownload(repository, upstreamURL string, index me if err != nil { return "", err } + return h.findChartURL(charts, digest, filename) +} + +func (h *HelmHandler) findChartURL(charts *helmChartLookup, digest, filename string) (string, error) { for _, chart := range charts.byDigest[digest] { if h.chartOnCooldown(chart.name, chart.created) { continue @@ -376,6 +402,37 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh return lookup, nil } +func (h *HelmHandler) downloadChartLookup(r *http.Request, repository, upstreamURL string) (*helmChartLookup, error) { + key := "helm\x00" + h.indexCacheKey(repository, upstreamURL) + return publishIndex(h.proxy, key, func() metadataResult { + return h.fetchIndex(r.Context(), repository, upstreamURL) + }, func(res metadataResult) (*helmChartLookup, error) { + if res.err != nil { + return nil, res.err + } + return h.chartLookup(repository, upstreamURL, res) + }) +} + +func (h *HelmHandler) withholdChart(w http.ResponseWriter, charts *helmChartLookup, digest, filename string) bool { + withheld := false + for _, chart := range charts.byDigest[digest] { + for _, download := range chart.downloads { + if download.filename != filename { + continue + } + if !h.chartOnCooldown(chart.name, chart.created) { + return false + } + withheld = true + } + } + if withheld { + http.Error(w, ErrVersionNotFound.Error(), http.StatusNotFound) + } + return withheld +} + func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { return !created.IsZero() && h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() && !h.proxy.cooldown().IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created) diff --git a/internal/handler/hex.go b/internal/handler/hex.go index f2a93ad..d394c4d 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -78,6 +78,7 @@ func (h *HexHandler) handleDownload(w http.ResponseWriter, r *http.Request) { parseErr: "could not parse tarball filename", fetchErr: "failed to fetch package", parse: h.parseTarballFilename, + publishTime: h.publishTime, }) } @@ -241,6 +242,10 @@ func parseHexReleases(body []byte) (publishTimes, error) { return times, nil } +func (h *HexHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, bool, error) { + return h.proxy.lookupPublishTime(r.Context(), h.timedPackage(name), version, h.packageVersions) +} + // signedPayload decompresses a gzipped Signed message, with the metadata // limit applied to the decompressed size, and returns its payload. func (h *HexHandler) signedPayload(gzippedData []byte) ([]byte, error) { diff --git a/internal/handler/homebrew.go b/internal/handler/homebrew.go index 0af67af..4e40d02 100644 --- a/internal/handler/homebrew.go +++ b/internal/handler/homebrew.go @@ -64,7 +64,7 @@ func (h *HomebrewHandler) Routes() http.Handler { if strings.HasPrefix(requestPath, "analytics/") { acceptEncoding = "identity" } - h.proxy.proxyCachedWithEncoding(w, r, upstreamURL, homebrewMetadataEcosystem, homebrewMetadataCacheKey(requestPath, r.URL.RawQuery), acceptEncoding, "*/*") + h.proxy.proxyCachedWithEncoding(w, r, upstreamURL, homebrewMetadataEcosystem, homebrewMetadataCacheKey(requestPath, r.URL.RawQuery), acceptEncoding, nil, "*/*") }) } diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 6c6a647..50d3046 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "io" "net/http" "path" "strings" @@ -175,32 +176,51 @@ func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, so return } - body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, urlPath) - if err != nil { - h.writeMetadataError(w, err) + // The checksums of maven-metadata.xml are computed from the cached + // metadata, as with cooldown, so they never describe another deploy. + if suffix, ok := strings.CutPrefix(path.Base(urlPath), mavenMetadataFile); ok && mavenChecksums[suffix] != nil { + res, _ := h.fetchMetadata(r.Context(), source, path.Join(path.Dir(urlPath), mavenMetadataFile)) + var body []byte + if res.err == nil { + body, res.err = h.proxy.decodedMetadataBody(res) + } + if res.err != nil { + h.writeMetadataError(w, res.err) + return + } + sum := mavenChecksums[suffix]() + _, _ = sum.Write(body) + w.Header().Set(headerContentType, "text/plain") + markStale(w, res.stale) + _, _ = io.WriteString(w, hex.EncodeToString(sum.Sum(nil))) return } - h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType) + res, _ := h.fetchMetadata(r.Context(), source, urlPath) + if res.err != nil { + h.writeMetadataError(w, res.err) + return + } + writeMetadataResult(w, r, res) } // fetchMetadata fetches a metadata file from the source's upstream, or from // its fallback when the upstream doesn't have it, and returns it with the // key it is cached under. -func (h *MavenHandler) fetchMetadata(ctx context.Context, source mavenSource, urlPath string) (body []byte, contentType, cacheKey string, err error) { - cacheKey = source.metadataCacheKey(urlPath) +func (h *MavenHandler) fetchMetadata(ctx context.Context, source mavenSource, urlPath string) (metadataResult, string) { + cacheKey := source.metadataCacheKey(urlPath) upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) - body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, upstreamURL, "*/*") - if err != nil && source.fallbackURL != "" { - if errors.Is(err, ErrUpstreamNotFound) { + res := h.proxy.lookupMetadata(ctx, "maven", cacheKey, upstreamURL, "*/*", "", nil) + if res.err != nil && source.fallbackURL != "" { + if errors.Is(res.err, ErrUpstreamNotFound) { pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal", "path", urlPath) - body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, pluginPortalURL, "*/*") + res = h.proxy.lookupMetadata(ctx, "maven", cacheKey, pluginPortalURL, "*/*", "", nil) } } - return body, contentType, cacheKey, err + return res, cacheKey } func (h *MavenHandler) writeMetadataError(w http.ResponseWriter, err error) { @@ -221,6 +241,13 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so // e.g., com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar group, artifact, version, filename := h.parsePath(urlPath) if artifact == "" { + if strings.Count(urlPath, "/") >= minMavenParts-1 { + // A dot or colon in a group directory, or a colon in the + // artifact, which the Maven layout never has. Proxied, the + // file would skip the denylist, scan and cooldown. + http.NotFound(w, r) + return + } h.proxyUpstream(w, r, source, urlPath) return } @@ -242,13 +269,21 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) cacheFilename := source.cacheFilename(filename) - result, err := h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, upstreamURL) + // Each build replaces a SNAPSHOT file named without a timestamp, so a + // GET fetches it again rather than serving the cached copy. HEAD is + // answered from cache. + get := h.proxy.GetOrFetchArtifactFromURL + if h.isMutableSnapshotFile(urlPath) && r.Method != http.MethodHead { + get = h.proxy.refetchArtifactFromURL + } + + result, err := get(ctx, "maven", name, version, cacheFilename, upstreamURL) if err != nil && source.fallbackURL != "" { if errors.Is(err, ErrUpstreamNotFound) { pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven artifact not found in primary upstream, trying Gradle Plugin Portal", "group", group, "artifact", artifact, "version", version, "filename", filename) - result, err = h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, pluginPortalURL) + result, err = get(ctx, "maven", name, version, cacheFilename, pluginPortalURL) } } if err != nil { @@ -266,6 +301,7 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so // parsePath extracts Maven coordinates from a URL path. // e.g., "com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar" // -> ("com.google.guava", "guava", "32.1.3-jre", "guava-32.1.3-jre.jar") +// A path that isn't in the Maven layout returns empty strings. func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, filename string) { parts := strings.Split(urlPath, "/") if len(parts) < minMavenParts { @@ -276,6 +312,18 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file version = parts[len(parts)-2] artifact = parts[len(parts)-3] groupParts := parts[:len(parts)-3] + // The Maven layout splits a group at its dots, so a group directory + // never holds a dot or is empty. Such a path would spell the same group + // as another and share its cache entries, so it is not parsed. Nor is a + // colon in the group or artifact, which are cached as group:artifact. + for _, part := range groupParts { + if part == "" || strings.ContainsAny(part, ".:") { + return "", "", "", "" + } + } + if strings.Contains(artifact, ":") { + return "", "", "", "" + } group = strings.Join(groupParts, ".") return group, artifact, version, filename @@ -301,14 +349,30 @@ func (h *MavenHandler) isArtifactFile(filename string) bool { // artifact and version of its directory, {artifact}-{version}..., as Maven // names every file it publishes, whatever the extension. A SNAPSHOT's // files carry a timestamp in place of "SNAPSHOT". +// +// The group isn't checked, so a file under a group directory outside the +// Maven layout still goes to handleDownload, which answers 404. func (h *MavenHandler) isCoordinateFile(urlPath string) bool { - _, artifact, version, filename := h.parsePath(urlPath) + parts := strings.Split(urlPath, "/") + if len(parts) < minMavenParts { + return false + } + filename, version, artifact := parts[len(parts)-1], parts[len(parts)-2], parts[len(parts)-3] if artifact == "" || version == "" { return false } return strings.HasPrefix(filename, artifact+"-"+strings.TrimSuffix(version, "SNAPSHOT")) } +// isMutableSnapshotFile reports whether urlPath is a SNAPSHOT file named +// with "SNAPSHOT" rather than a build timestamp (lib-1.0-SNAPSHOT.jar), which +// each new build replaces. Timestamped SNAPSHOT files never change. +func (h *MavenHandler) isMutableSnapshotFile(urlPath string) bool { + _, artifact, version, filename := h.parsePath(urlPath) + return artifact != "" && strings.HasSuffix(version, mavenSnapshot) && + strings.HasPrefix(filename, artifact+"-"+version) +} + // isVersionListingFile matches maven-metadata.xml and its checksums. The // listing changes with every release, so it is cached for the metadata TTL // rather than kept like an artifact. @@ -328,7 +392,21 @@ func (h *MavenHandler) isChecksumFile(filename string) bool { return false } +// mavenForwardHeaders lets clients resume artifacts proxied without caching. +var mavenForwardHeaders = []string{"Range", "If-Range"} + // proxyUpstream forwards a request to the source's upstream without caching. func (h *MavenHandler) proxyUpstream(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { - h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, nil) + if source.fallbackURL == "" { + h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, mavenForwardHeaders) + return + } + // As for cached files, a file the primary upstream lacks is looked for + // on the Gradle Plugin Portal. + resp, err := h.proxy.upstreamRequest(r, source.upstreamURL+"/"+urlPath, mavenForwardHeaders) + if err == nil && resp.StatusCode == http.StatusNotFound { + _ = resp.Body.Close() + resp, err = h.proxy.upstreamRequest(r, source.fallbackURL+"/"+urlPath, mavenForwardHeaders) + } + h.proxy.relayUpstream(w, r, resp, err) } diff --git a/internal/handler/maven_consistency_test.go b/internal/handler/maven_consistency_test.go new file mode 100644 index 0000000..e548447 --- /dev/null +++ b/internal/handler/maven_consistency_test.go @@ -0,0 +1,157 @@ +package handler + +import ( + "crypto/sha1" //nolint:gosec // Maven publishes SHA-1 checksums. + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/registries/fetch" +) + +// mavenGenerations serves every path with the current generation's bytes, +// and a matching .sha1 for maven-metadata.xml, counting requests per path. +type mavenGenerations struct { + gen atomic.Int32 + requests map[string]*atomic.Int32 + // down makes every request fail with 502. + down atomic.Bool +} + +func (m *mavenGenerations) body(path string) string { + return path + " generation " + string(rune('0'+m.gen.Load())) +} + +func newMavenGenerations(t *testing.T, paths ...string) (*mavenGenerations, http.Handler) { + t.Helper() + m := &mavenGenerations{requests: map[string]*atomic.Int32{}} + for _, p := range paths { + m.requests[p] = &atomic.Int32{} + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if n := m.requests[r.URL.Path]; n != nil { + n.Add(1) + } + if m.down.Load() { + http.Error(w, "unavailable", http.StatusBadGateway) + return + } + const metadata = "/com/example/lib/maven-metadata.xml" + if r.URL.Path == metadata+".sha1" { + sum := sha1.Sum([]byte(m.body(metadata))) //nolint:gosec // as above + _, _ = io.WriteString(w, hex.EncodeToString(sum[:])) + return + } + _, _ = io.WriteString(w, m.body(r.URL.Path)) + })) + t.Cleanup(srv.Close) + + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = srv.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + return m, NewMavenHandler(proxy, "http://localhost", srv.URL, srv.URL).Routes() +} + +// TestMavenMetadataChecksumMatchesServedMetadata checks that the checksum +// files of maven-metadata.xml describe the metadata the proxy serves, even +// when upstream has deployed since it was cached. +func TestMavenMetadataChecksumMatchesServedMetadata(t *testing.T) { + m, h := newMavenGenerations(t) + const metadata = "/com/example/lib/maven-metadata.xml" + + served := getMaven(h, metadata).Body.String() + m.gen.Store(1) + w := getMaven(h, metadata+".sha1") + sum := sha1.Sum([]byte(served)) //nolint:gosec // as above + if w.Code != http.StatusOK || w.Body.String() != hex.EncodeToString(sum[:]) { + t.Errorf(".sha1 = %d %q, want the sha1 of the served metadata %x", w.Code, w.Body, sum) + } +} + +// TestMavenNonUniqueSnapshotNotKept checks that a SNAPSHOT file without a +// timestamp, which each build replaces, is not served from cache for good, +// while a timestamped one is. +func TestMavenNonUniqueSnapshotNotKept(t *testing.T) { + const ( + snapshot = "/com/example/lib/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar" + timestamped = "/com/example/lib/1.0-SNAPSHOT/lib-1.0-20240101.120000-1.jar" + ) + m, h := newMavenGenerations(t, timestamped) + + getMaven(h, snapshot) + getMaven(h, snapshot+".sha1") + getMaven(h, timestamped) + m.gen.Store(1) + if w := getMaven(h, snapshot); w.Body.String() != m.body(snapshot) { + t.Errorf("SNAPSHOT jar = %q, want the new build %q", w.Body, m.body(snapshot)) + } + if w := getMaven(h, snapshot+".sha1"); w.Body.String() != m.body(snapshot+".sha1") { + t.Errorf("SNAPSHOT jar checksum = %q, want the new build's", w.Body) + } + if w := getMaven(h, timestamped); w.Code != http.StatusOK || m.requests[timestamped].Load() != 1 { + t.Errorf("timestamped SNAPSHOT jar fetched %d times, want once", m.requests[timestamped].Load()) + } +} + +// TestMavenMutableSnapshotKeepsPolicyAndFallback checks that a SNAPSHOT file +// fetched again for each request still goes through the denylist, and falls +// back to the Gradle Plugin Portal when the primary upstream lacks it. +func TestMavenMutableSnapshotKeepsPolicyAndFallback(t *testing.T) { + const snapshot = "/com/example/lib/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar" + primary := httptest.NewServer(http.NotFoundHandler()) + t.Cleanup(primary.Close) + portal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "portal build") + })) + t.Cleanup(portal.Close) + + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.HTTPClient = primary.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(primary.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewMavenHandler(proxy, "http://localhost", primary.URL, portal.URL).Routes() + + if w := getMaven(h, snapshot); w.Code != http.StatusOK || w.Body.String() != "portal build" { + t.Errorf("SNAPSHOT from the fallback = %d %q, want 200 portal build", w.Code, w.Body) + } + setTestDenylist(t, proxy, "pkg:maven/com.example/lib@1.0-SNAPSHOT") + if w := getMaven(h, snapshot); w.Code != http.StatusForbidden { + t.Errorf("denied SNAPSHOT = %d, want 403", w.Code) + } +} + +// TestMavenMutableSnapshotServesCacheWhenUpstreamFails checks that a +// SNAPSHOT file fetched again for each request is still served from cache +// while upstream is down, and that HEAD does not download it again. +func TestMavenMutableSnapshotServesCacheWhenUpstreamFails(t *testing.T) { + const snapshot = "/com/example/lib/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar" + m, h := newMavenGenerations(t, snapshot) + + built := getMaven(h, snapshot).Body.String() + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodHead, snapshot, nil)) + if w.Code != http.StatusOK || m.requests[snapshot].Load() != 1 { + t.Errorf("HEAD = %d after %d upstream requests, want 200 from cache", w.Code, m.requests[snapshot].Load()) + } + + m.down.Store(true) + if w := getMaven(h, snapshot); w.Code != http.StatusOK || w.Body.String() != built { + t.Errorf("SNAPSHOT while upstream is down = %d %q, want the cached build %q", w.Code, w.Body, built) + } + m.down.Store(false) + m.gen.Store(1) + if w := getMaven(h, snapshot); w.Body.String() != m.body(snapshot) { + t.Errorf("SNAPSHOT once upstream is back = %q, want the new build", w.Body) + } +} diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index 4da839e..6155e0d 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -242,11 +242,12 @@ func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Requ return false } metadataPath := path.Join(path.Dir(urlPath), mavenMetadataFile) - body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, metadataPath) - if err != nil { - h.writeMetadataError(w, err) + res, cacheKey := h.fetchMetadata(r.Context(), source, metadataPath) + if res.err != nil { + h.writeMetadataError(w, res.err) return true } + body, contentType := res.body, res.contentType m, err := h.proxy.cachedRewrite(r.Context(), "maven", h.proxyURL, cacheKey, body, "", func(in []byte) ([]byte, []string, error) { // The rewrite is shared by requests waiting on it and cached, so it @@ -273,7 +274,7 @@ func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Requ if contentType == "" || contentType == contentTypeJSON { contentType = "text/xml" } - markStale(w, h.proxy.lookupCachedMeta("maven", cacheKey).stale) + markStale(w, res.stale) serveRewrittenMetadataAs(w, r, m, contentType) case mavenChecksums[suffix] != nil: sum := mavenChecksums[suffix]() diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go index 8272f37..e9b2fcf 100644 --- a/internal/handler/maven_repositories_test.go +++ b/internal/handler/maven_repositories_test.go @@ -7,6 +7,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "github.com/git-pkgs/registries/fetch" ) @@ -137,3 +138,88 @@ func TestMavenRepositoryRepointedServesTheNewUpstream(t *testing.T) { } } } + +// TestMavenDottedGroupDirectoryNotCachedAsGroup checks that a file whose +// group directories contain dots or colons, which the Maven layout never +// has, is not served: it would either share the cache entry of the group +// the dots spell or be proxied past the denylist, scan and cooldown. +func TestMavenDottedGroupDirectoryNotCachedAsGroup(t *testing.T) { + const standard = "/org/foo/bar/1.0/bar-1.0.jar" + odd := []string{ + "/org.foo/bar/1.0/bar-1.0.jar", "/org.foo/bar/1.0/bar-1.0-linux.exe", + "/org/foo:x/bar/1.0/bar-1.0.jar", "/org/foo/b:ar/1.0/b:ar-1.0.jar", + } + central, requests := mavenRepoUpstream(t, "central", append([]string{standard}, odd...)...) + portal, _ := mavenRepoUpstream(t, "portal") + var restricted []string + for _, p := range odd { + restricted = append(restricted, "/restricted"+p) + } + grails, grailsRequests := mavenRepoUpstream(t, "grails", restricted...) + h := newMavenRepositoriesHandler(t, central, portal, grails) + + if w := getMaven(h, standard); w.Code != http.StatusOK || w.Body.String() != "central "+standard { + t.Errorf("GET %s = %d %q", standard, w.Code, w.Body.String()) + } + before := requests.Load() + grailsRequests.Load() + for _, p := range odd { + for _, path := range []string{p, "/grails" + p} { + if w := getMaven(h, path); w.Code != http.StatusNotFound { + t.Errorf("GET %s = %d %q, want 404", path, w.Code, w.Body.String()) + } + } + } + if n := requests.Load() + grailsRequests.Load() - before; n != 0 { + t.Errorf("%d upstream requests for paths outside the Maven layout", n) + } +} + +// TestMavenUncachedPathFallsBackToPluginPortal checks that a path proxied +// without caching, such as a catalog, still tries the Gradle Plugin Portal +// when the primary upstream lacks it. +func TestMavenUncachedPathFallsBackToPluginPortal(t *testing.T) { + const uncached = "/org/foo/archetype-catalog.xml" + central, _ := mavenRepoUpstream(t, "central") + portal, _ := mavenRepoUpstream(t, "portal", uncached) + grails, _ := mavenRepoUpstream(t, "grails") + h := newMavenRepositoriesHandler(t, central, portal, grails) + + if w := getMaven(h, uncached); w.Code != http.StatusOK || w.Body.String() != "portal "+uncached { + t.Errorf("GET %s = %d %q, want the Plugin Portal's copy", uncached, w.Code, w.Body.String()) + } +} + +func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { + const uncached = "/org/foo/archetype-catalog.xml" + ranged := func() *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"v2"`) + http.ServeContent(w, r, "archetype-catalog.xml", time.Time{}, strings.NewReader("0123456789")) + })) + t.Cleanup(srv.Close) + return srv + } + central, grails := ranged(), ranged() + portal, _ := mavenRepoUpstream(t, "portal") + h := newMavenRepositoriesHandler(t, central, portal, grails) + + for _, p := range []string{uncached, "/grails" + uncached} { + req := httptest.NewRequest(http.MethodGet, p, nil) + req.Header.Set("Range", "bytes=2-4") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusPartialContent || w.Body.String() != "234" { + t.Errorf("GET %s with Range = %d %q, want 206 %q", p, w.Code, w.Body.String(), "234") + } + + // A Range for an older copy gets the whole current file. + req = httptest.NewRequest(http.MethodGet, p, nil) + req.Header.Set("Range", "bytes=2-4") + req.Header.Set("If-Range", `"v1"`) + w = httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusOK || w.Body.String() != "0123456789" { + t.Errorf("GET %s with stale If-Range = %d %q, want 200 with the whole file", p, w.Code, w.Body.String()) + } + } +} diff --git a/internal/handler/maven_test.go b/internal/handler/maven_test.go index 9d34b09..14cb49f 100644 --- a/internal/handler/maven_test.go +++ b/internal/handler/maven_test.go @@ -31,6 +31,15 @@ func TestMavenParsePath(t *testing.T) { "short/path", "", "", "", "", }, + // Group directories never hold dots or are empty in the Maven + // layout; these would spell the same group as org/foo. + {"org.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + {"org/.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + {"org//foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + // Cache identity is group:artifact, so a colon in either would + // let org:foo/bar and org/foo:bar share one entry. + {"org:foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + {"org/foo:bar/1.0/bar-1.0.jar", "", "", "", ""}, } for _, tt := range tests { diff --git a/internal/handler/metadata_coalesce_test.go b/internal/handler/metadata_coalesce_test.go index af8e2eb..26e0761 100644 --- a/internal/handler/metadata_coalesce_test.go +++ b/internal/handler/metadata_coalesce_test.go @@ -247,3 +247,39 @@ func TestCoalescedMetadataMiss_ServesRowCommittedSinceLookup(t *testing.T) { t.Errorf("upstream requests = %d, want 1 (the priming fetch only)", got) } } + +// TestFetchOrCacheMetadata_DetachCapFallsBackToClientContext checks the cap on +// metadata fetches that outlive their caller: with every slot taken, a caller +// that leaves ends its fetch, so clients that request documents and hang up +// can't leave any number of fetches running. +func TestFetchOrCacheMetadata_DetachCapFallsBackToClientContext(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + defer close(u.release) + p := metadataTestProxy(t, u) + + for range maxDetachedMetadataFetches { + if !p.acquireMetadataDetachSlot() { + t.Fatal("could not take a free metadata detach slot") + } + } + ctx, cancel := context.WithCancel(context.Background()) + call := startMetadataCalls(p, ctx, 1, u.URL, contentTypeJSON) + <-u.entered + cancel() + + select { + case c := <-call[0]: + if !errors.Is(c.err, context.Canceled) { + t.Errorf("err = %v, want context.Canceled once the slots are used up", c.err) + } + case <-time.After(5 * time.Second): + t.Fatal("the fetch kept running after its caller left, with no slot free") + } + + for range maxDetachedMetadataFetches { + p.releaseMetadataDetachSlot() + } + if !p.acquireMetadataDetachSlot() { + t.Error("slots not available again after release") + } +} diff --git a/internal/handler/metadata_consistency_test.go b/internal/handler/metadata_consistency_test.go new file mode 100644 index 0000000..3e0220a --- /dev/null +++ b/internal/handler/metadata_consistency_test.go @@ -0,0 +1,324 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "context" + "database/sql" + "errors" + "io" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// proxyCachedRequest runs one ProxyCached request for key "pkg". +func proxyCachedRequest(proxy *Proxy, upstreamURL string, header http.Header) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/pkg", nil) + for k, v := range header { + r.Header[k] = v + } + proxy.ProxyCached(w, r, upstreamURL+"/pkg", "test", "pkg") + return w +} + +// TestFetchedMetadataValidatorsDescribeBody checks that a document fetched +// fresh but not stored is served with upstream's validators, not those of the +// older row still in the cache. +func TestFetchedMetadataValidatorsDescribeBody(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + if requests.Add(1) == 1 { + w.Header().Set(headerETag, `"v1"`) + _, _ = w.Write([]byte(`{"v":1}`)) + return + } + w.Header().Set(headerETag, `"v2"`) + _, _ = w.Write([]byte(`{"v":2}`)) + })) + t.Cleanup(upstream.Close) + + proxy, _, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Millisecond + proxy.HTTPClient = upstream.Client() + + if w := proxyCachedRequest(proxy, upstream.URL, nil); w.Code != http.StatusOK { + t.Fatalf("first request: %d", w.Code) + } + time.Sleep(5 * time.Millisecond) + store.mu.Lock() + store.storeErr = errors.New("disk full") + store.mu.Unlock() + + w := proxyCachedRequest(proxy, upstream.URL, nil) + if w.Code != http.StatusOK || w.Body.String() != `{"v":2}` { + t.Fatalf("second request = %d %q, want v2", w.Code, w.Body) + } + if got := w.Header().Get(headerETag); got != `"v2"` { + t.Errorf("ETag = %q, want %q", got, `"v2"`) + } + if got := w.Header().Get("Warning"); got != "" { + t.Errorf("Warning = %q on a fresh document", got) + } + + // A client holding v1 must get v2, not a 304. + w = proxyCachedRequest(proxy, upstream.URL, http.Header{"If-None-Match": {`"v1"`}}) + if w.Code != http.StatusOK || w.Body.String() != `{"v":2}` { + t.Errorf("conditional request = %d %q, want 200 v2", w.Code, w.Body) + } +} + +// TestStaleMetadataMarkedWithoutTTL checks that the stale fallback carries +// the Warning header with metadata_ttl 0, where every request revalidates. +func TestStaleMetadataMarkedWithoutTTL(t *testing.T) { + upstream := upstreamThen(t, `{"v":1}`, http.StatusBadGateway) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 + proxy.HTTPClient = upstream.Client() + + if w := proxyCachedRequest(proxy, upstream.URL, nil); w.Code != http.StatusOK { + t.Fatalf("first request: %d", w.Code) + } + w := proxyCachedRequest(proxy, upstream.URL, nil) + if w.Code != http.StatusOK || w.Body.String() != `{"v":1}` { + t.Fatalf("fallback = %d %q", w.Code, w.Body) + } + if got := w.Header().Get("Warning"); got != staleWarning { + t.Errorf("Warning = %q, want %q", got, staleWarning) + } +} + +// seedMetadataRow stores {"v":1} for key "pkg" with ETag "v1" and a row past +// its TTL. +func seedMetadataRow(t *testing.T, db *database.DB, store *mockStorage) string { + t.Helper() + const body, etag = `{"v":1}`, `"v1"` + path := metadataStoragePath("test", "pkg") + store.mu.Lock() + store.files[path] = []byte(body) + store.mu.Unlock() + if err := db.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "test", + Name: "pkg", + StoragePath: path, + ETag: sql.NullString{String: etag, Valid: true}, + ContentType: sql.NullString{String: contentTypeJSON, Valid: true}, + ContentDigest: sql.NullString{String: "sha256:" + sha256Hex(body), Valid: true}, + Size: sql.NullInt64{Int64: int64(len(body)), Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, + }); err != nil { + t.Fatal(err) + } + return path +} + +// v2Upstream answers If-None-Match "v1" with 304 after running onNotModified, +// and any other request with v2. +func v2Upstream(t *testing.T, onNotModified func()) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("If-None-Match") == `"v1"` { + onNotModified() + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set(headerContentType, contentTypeJSON) + w.Header().Set(headerETag, `"v2"`) + _, _ = w.Write([]byte(`{"v":2}`)) + })) + t.Cleanup(srv.Close) + return srv +} + +// TestNotModifiedChecksStoredBytes checks that a 304 for a row whose blob has +// meanwhile been replaced (by another instance sharing the store) does not +// serve the new bytes under the old ETag. +func TestNotModifiedChecksStoredBytes(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Minute + path := seedMetadataRow(t, db, store) + upstream := v2Upstream(t, func() { + // The other instance has written v2 but not yet its row. + store.mu.Lock() + store.files[path] = []byte(`{"v":2}`) + store.mu.Unlock() + }) + proxy.HTTPClient = upstream.Client() + + w := proxyCachedRequest(proxy, upstream.URL, nil) + if w.Code != http.StatusOK || w.Body.String() != `{"v":2}` { + t.Fatalf("response = %d %q", w.Code, w.Body) + } + if got := w.Header().Get(headerETag); got != `"v2"` { + t.Errorf("ETag = %q, want %q", got, `"v2"`) + } + row, err := db.GetMetadataCache("test", "pkg") + if err != nil || row == nil { + t.Fatal(row, err) + } + if row.ETag.String != `"v2"` || row.ContentDigest.String != "sha256:"+sha256Hex(`{"v":2}`) { + t.Errorf("row = etag %q digest %q, want v2's", row.ETag.String, row.ContentDigest.String) + } +} + +// TestNotModifiedAfterRowReplaced checks that a 304 for a row another +// instance has replaced since it was read does not store the old document +// back over the new one. +func TestNotModifiedAfterRowReplaced(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Minute + path := seedMetadataRow(t, db, store) + upstream := v2Upstream(t, func() { + // The other instance has recorded v2's row; its blob is not written + // yet. + _ = db.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "test", + Name: "pkg", + StoragePath: path, + ETag: sql.NullString{String: `"v2"`, Valid: true}, + ContentType: sql.NullString{String: contentTypeJSON, Valid: true}, + ContentDigest: sql.NullString{String: "sha256:" + sha256Hex(`{"v":2}`), Valid: true}, + Size: sql.NullInt64{Int64: 7, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + }) + }) + proxy.HTTPClient = upstream.Client() + + res := proxy.fetchMetadataDocument(context.Background(), "test", "pkg", upstream.URL+"/pkg", contentTypeJSON, nil) + if res.err != nil || string(res.body) != `{"v":2}` { + t.Fatalf("fetch = %q, %v; want v2", res.body, res.err) + } + row, err := db.GetMetadataCache("test", "pkg") + if err != nil || row == nil { + t.Fatal(row, err) + } + if row.ETag.String != `"v2"` { + t.Errorf("row etag = %q, want %q", row.ETag.String, `"v2"`) + } +} + +// TestNotModifiedRejectedBodyNotRefreshed checks that a 304 for a stored +// document validate rejects does not mark that document fresh. +func TestNotModifiedRejectedBodyNotRefreshed(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Minute + seedMetadataRow(t, db, store) + before, err := db.GetMetadataCache("test", "pkg") + if err != nil || before == nil { + t.Fatal(before, err) + } + upstream := v2Upstream(t, func() {}) + proxy.HTTPClient = upstream.Client() + + reject := func(body []byte) error { + if string(body) == `{"v":1}` { + return errors.New("damaged") + } + return nil + } + proxy.fetchMetadataDocument(context.Background(), "test", "pkg", upstream.URL+"/pkg", contentTypeJSON, reject) + row, err := db.GetMetadataCache("test", "pkg") + if err != nil || row == nil { + t.Fatal(row, err) + } + if !row.FetchedAt.Time.Equal(before.FetchedAt.Time) { + t.Errorf("rejected v1 row fetched_at moved from %v to %v", before.FetchedAt.Time, row.FetchedAt.Time) + } +} + +// openHookStorage runs onOpen before the first Open. +type openHookStorage struct { + *mockStorage + once sync.Once + onOpen func() +} + +func (s *openHookStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) { + s.once.Do(s.onOpen) + return s.mockStorage.Open(ctx, path) +} + +// TestStaleMetadataChecksStoredBytes checks that the stale fallback does not +// serve a blob another instance replaced after the row was read under that +// row's ETag. +func TestStaleMetadataChecksStoredBytes(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + t.Cleanup(upstream.Close) + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 + proxy.HTTPClient = upstream.Client() + path := seedMetadataRow(t, db, store) + proxy.Storage = &openHookStorage{mockStorage: store, onOpen: func() { + // The other instance has written v2 but not yet its row. + store.mu.Lock() + store.files[path] = []byte(`{"v":2}`) + store.mu.Unlock() + }} + + // No row describes v2 yet, so its encoding is unknown: nothing is served. + w := proxyCachedRequest(proxy, upstream.URL, http.Header{"If-None-Match": {`"v1"`}}) + if w.Code < http.StatusInternalServerError || w.Header().Get(headerETag) == `"v1"` { + t.Errorf("response = %d ETag %q for v2's bytes, want an error without v1 validators", w.Code, w.Header().Get(headerETag)) + } +} + +// TestStaleMetadataRereadsReplacedRow checks that when the stored bytes no +// longer match the row, the stale fallback reads the row again and serves +// the blob with the encoding and validators of the row that describes it. +func TestStaleMetadataRereadsReplacedRow(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + t.Cleanup(upstream.Close) + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 + proxy.HTTPClient = upstream.Client() + path := seedMetadataRow(t, db, store) + var gz bytes.Buffer + zw := gzip.NewWriter(&gz) + _, _ = zw.Write([]byte(`{"v":2}`)) + _ = zw.Close() + v2 := gz.Bytes() + proxy.Storage = &openHookStorage{mockStorage: store, onOpen: func() { + // The other instance stores v2 gzipped, then its row. + store.mu.Lock() + store.files[path] = v2 + store.mu.Unlock() + if err := db.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "test", + Name: "pkg", + StoragePath: path, + ETag: sql.NullString{String: `"v2"`, Valid: true}, + ContentType: sql.NullString{String: contentTypeJSON, Valid: true}, + ContentEncoding: sql.NullString{String: "gzip", Valid: true}, + ContentDigest: sql.NullString{String: "sha256:" + sha256Hex(string(v2)), Valid: true}, + Size: sql.NullInt64{Int64: int64(len(v2)), Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, + }); err != nil { + t.Error(err) + } + }} + + w := proxyCachedRequest(proxy, upstream.URL, http.Header{"Accept-Encoding": {"gzip"}}) + if w.Code != http.StatusOK || w.Header().Get("Content-Encoding") != "gzip" || + w.Header().Get(headerETag) != `"v2"` || !bytes.Equal(w.Body.Bytes(), v2) { + t.Errorf("response = %d, Content-Encoding %q, ETag %q; want v2 gzipped under its own row", + w.Code, w.Header().Get("Content-Encoding"), w.Header().Get(headerETag)) + } +} diff --git a/internal/handler/metadata_immutable.go b/internal/handler/metadata_immutable.go index 691fbc1..214077c 100644 --- a/internal/handler/metadata_immutable.go +++ b/internal/handler/metadata_immutable.go @@ -40,9 +40,8 @@ func (p *Proxy) proxyImmutable(w http.ResponseWriter, r *http.Request, upstreamU p.writeMetadataFetchError(w, res.err) return } - cm := p.lookupCachedMeta(ecosystem, cacheKey) - cm.stale = false - writeMetadataBody(w, r, cm, res.body, res.contentType, res.contentEncoding) + res.stale = false + writeMetadataResult(w, r, res) } // fetchImmutableMetadata returns an immutable metadata document from the @@ -74,7 +73,7 @@ func (p *Proxy) fetchImmutableMetadata(ctx context.Context, ecosystem, cacheKey, return metadataResult{err: err} } digest := p.cacheMetadataBlob(fetchCtx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} + return fetchedResult(meta, digest) }) return res } diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go new file mode 100644 index 0000000..fc16d43 --- /dev/null +++ b/internal/handler/metadata_stored.go @@ -0,0 +1,313 @@ +package handler + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/http" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" +) + +// A metadata document can be rewritten from storage instead of from memory: +// storedMetadataDocument streams it from upstream into storage, and the +// rewrite reads it back as often as it needs. Nothing holds the whole +// document, so its size is not limited by metadata_max_size or by memory. + +// errMetadataNotStored is returned when metadata is not cached in storage, so +// a document has to be read into memory. +var errMetadataNotStored = errors.New("metadata is not cached in storage") + +// errStoredMetadataChanged is returned when a stored document no longer has +// the digest its cache row records: a fetch replaced it while it was read. +var errStoredMetadataChanged = errors.New("stored metadata changed while it was read") + +// storedMetadataDocument returns the cache row of a metadata document stored +// uncompressed with its digest: the stored copy while it is within its TTL, +// or else the document fetched from upstream and stored, or the stored copy +// when upstream fails. The document is streamed from upstream into storage +// without being held in memory, so a fetch takes no metadata_max_size limit. +// Concurrent calls for one document share a fetch, and instances sharing the +// database take turns as for other metadata. +// +// validate, when not nil, checks a fetched document before it replaces the +// stored one; a document it rejects is not stored, and the stored copy is +// served as when upstream fails. It does not check a stored copy within its +// TTL. +// +// It returns errMetadataNotStored when the proxy doesn't cache metadata or +// storing the document failed, and the caller reads it into memory instead. +func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error) (*database.MetadataCacheEntry, error) { + res := p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, "") + return res.entry, res.err +} + +// storedMetadata implements storedMetadataDocument, and reports in the +// result whether the row was read from cache or served stale. A row with +// the digest gone, one whose stored copy was missing or out of date when +// read, is not served or revalidated, within its TTL or not, and the +// document is fetched again. +func (p *Proxy) storedMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone string) metadataResult { + if !p.CacheMetadata || p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 { + return metadataResult{err: errMetadataNotStored} + } + if containsPathTraversal(cacheKey) { + return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} + } + if entry, fresh := p.storedMetadataEntryState(ctx, ecosystem, cacheKey, gone); fresh { + metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) + p.recordMetadataRead(entry) + return metadataResult{entry: entry, fromCache: true} + } + p.recordMetadataCacheMiss(ecosystem) + + key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, "", false) + "\x00stored" + res, _ := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { + entry, fresh := p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey, gone) + if fresh { + return metadataResult{entry: entry} + } + release, outcome, err := p.awaitFetchLease(fetchCtx, metrics.CacheKindMetadata, key, func() bool { + entry, fresh = p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey, gone) + return fresh + }) + if err != nil { + return metadataResult{err: err} + } + switch outcome { + case leaseStored: + return metadataResult{entry: entry} + case leaseFailed: + // Serve the stored copy, as the holder did, rather than trying + // the failing upstream again. + p.reloadUpstreamHolds() + if usableStoredMetadata(entry) { + return p.staleStoredMetadata(ecosystem, cacheKey, entry, errFetchFailedElsewhere) + } + case leaseFetch: + } + res := metadataResult{err: errSharedFetchAbandoned} + defer func() { release(res.err != nil || res.stale) }() + res = p.fetchStoredMetadata(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, validate, entry) + return res + }) + return res +} + +// storedMetadataEntryState reads the cache row of a document and reports +// whether it is within its TTL and usable by storedMetadataDocument. A row +// with the digest gone is returned as nil. +func (p *Proxy) storedMetadataEntryState(ctx context.Context, ecosystem, cacheKey, gone string) (*database.MetadataCacheEntry, bool) { + entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) + entry = p.migrateLegacyMetadataEntry(ctx, ecosystem, cacheKey, entry) + if gone != "" && entry != nil && entry.ContentDigest.String == gone { + return nil, false + } + fresh := usableStoredMetadata(entry) && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL + return entry, fresh +} + +// usableStoredMetadata reports whether entry describes a document stored +// uncompressed with its digest, which a rewrite can read back and check. +func usableStoredMetadata(entry *database.MetadataCacheEntry) bool { + return entry != nil && entry.ContentDigest.Valid && entry.ContentEncoding.String == "" +} + +// fetchStoredMetadata fetches a document from upstream into storage, +// revalidating entry when there is one, and falls back to entry when +// upstream fails or validate rejects what it sent. A failure to store is +// returned as it is, for the caller to read the document into memory. +func (p *Proxy) fetchStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, entry *database.MetadataCacheEntry) metadataResult { + revalidate := entry + if !usableStoredMetadata(entry) { + revalidate = nil + } + stored, err := p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, revalidate) + if errors.Is(err, errStale304) { + stored, err = p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, nil) + } + if err == nil { + return metadataResult{entry: stored} + } + // As in fetchMetadataFromUpstream, any upstream failure, a 404 included, + // falls back to the stored copy. + if !usableStoredMetadata(entry) || errors.Is(err, errMetadataNotStored) { + return metadataResult{err: err} + } + return p.staleStoredMetadata(ecosystem, cacheKey, entry, err) +} + +// staleStoredMetadata falls back to the stored document entry describes after +// the upstream fetch failed with err. +func (p *Proxy) staleStoredMetadata(ecosystem, cacheKey string, entry *database.MetadataCacheEntry, err error) metadataResult { + p.Logger.Warn("upstream metadata fetch failed, serving cached copy", + "ecosystem", ecosystem, "key", cacheKey, "error", err) + entry = p.currentMetadataEntry(ecosystem, cacheKey, entry) + if !usableStoredMetadata(entry) { + return metadataResult{err: fmt.Errorf("upstream failed and no usable cached metadata: %w", err)} + } + return metadataResult{entry: entry, stale: true, upstreamErr: err} +} + +// storeUpstreamMetadata fetches a document and streams it into storage, and +// returns its new cache row. On a 304 for entry it only moves the row's +// fetched_at on; errStale304 means the stored copy is gone or the row was +// replaced meanwhile, and the caller should fetch without validators. +// +// With validate, the document is staged under a path of its own and checked +// there first, so one validate rejects never replaces the stored copy. +func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, entry *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { + resp, err := p.requestUpstreamMetadata(ctx, upstreamURL, entry, accept, "") + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotModified && entry != nil { + if ok, err := p.Storage.Exists(ctx, entry.StoragePath); err != nil || !ok { + return nil, errStale304 + } + refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) + if err != nil || !refreshed { + return nil, errStale304 + } + return p.currentMetadataEntry(ecosystem, cacheKey, entry), nil + } + if err := upstreamMetadataStatus(resp); err != nil { + return nil, err + } + + meta := &upstreamMetadata{ + contentType: resp.Header.Get(headerContentType), + etag: resp.Header.Get(headerETag), + } + if meta.contentType == "" { + meta.contentType = contentTypeJSON + } + if lm := resp.Header.Get(headerLastModified); lm != "" { + meta.lastModified, _ = http.ParseTime(lm) + } + limit := p.MetadataStreamMaxSize + if limit <= 0 { + limit = defaultMetadataStreamMaxSize + } + // Go decompresses the body, since no Accept-Encoding was set, so it is + // stored uncompressed. + var body io.Reader = &limitedBody{r: upstreamBodyReader{resp.Body}, left: limit} + if validate != nil { + staged, err := p.stageMetadata(ctx, ecosystem, cacheKey, body, validate) + if err != nil { + return nil, err + } + defer func() { _ = staged.Close() }() + body = staged + } + if _, err := p.storeMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), body, meta); err != nil { + return nil, notStored("", err) + } + stored, err := p.DB.GetMetadataCache(ecosystem, cacheKey) + if err != nil || !usableStoredMetadata(stored) { + return nil, fmt.Errorf("reading the stored metadata row: %w", errors.Join(err, errMetadataNotStored)) + } + return stored, nil +} + +// stageMetadata stores body under a staging path of its own, checks it with +// validate, and returns it opened for reading, to be stored in place of the +// cached copy. The staged copy is deleted when the reader is closed. A +// failure to store it is returned as errMetadataNotStored. +func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, body io.Reader, validate func(io.Reader) error) (io.ReadCloser, error) { + path := metadataStoragePath(ecosystem, cacheKey) + ".staged-" + rand.Text() + discard := func() { + if err := p.Storage.Delete(context.WithoutCancel(ctx), path); err != nil { + p.Logger.Warn("failed to delete staged metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) + } + } + if _, _, err := p.Storage.Store(ctx, path, body); err != nil { + discard() + return nil, notStored("staging: ", err) + } + if err := readStored(func() (io.ReadCloser, error) { return p.Storage.Open(ctx, path) }, validate); err != nil { + discard() + return nil, err + } + staged, err := p.Storage.Open(ctx, path) + if err != nil { + discard() + return nil, fmt.Errorf("%w: reading staged copy: %w", errMetadataNotStored, err) + } + return &stagedReader{ReadCloser: staged, discard: discard}, nil +} + +// limitedBody fails with ErrMetadataTooLarge, as an upstream failure, once +// r has sent more than left bytes. +type limitedBody struct { + r io.Reader + left int64 +} + +func (l *limitedBody) Read(b []byte) (int, error) { + n, err := l.r.Read(b) + if l.left -= int64(n); l.left < 0 { + return 0, &upstreamBodyError{ErrMetadataTooLarge} + } + return n, err +} + +// notStored returns a failure to store a document as errMetadataNotStored, +// unless reading it from upstream failed. +func notStored(what string, err error) error { + if body := (*upstreamBodyError)(nil); errors.As(err, &body) { + return body + } + return fmt.Errorf("%w: %s%w", errMetadataNotStored, what, err) +} + +// stagedReader deletes the staged copy it reads when closed. +type stagedReader struct { + io.ReadCloser + discard func() +} + +func (s *stagedReader) Close() error { + err := s.ReadCloser.Close() + s.discard() + return err +} + +// openStoredMetadata opens the document entry describes. Reading it to the +// end fails with errStoredMetadataChanged when its bytes don't have the +// digest entry records. +func (p *Proxy) openStoredMetadata(ctx context.Context, entry *database.MetadataCacheEntry) (io.ReadCloser, error) { + r, err := p.Storage.Open(ctx, entry.StoragePath) + if err != nil { + return nil, err + } + return &digestReader{r: r, h: sha256.New(), want: entry.ContentDigest.String}, nil +} + +// digestReader checks at the end of its reader that what it read has the +// digest want, in the form the metadata cache records it. +type digestReader struct { + r io.ReadCloser + h hash.Hash + want string +} + +func (d *digestReader) Read(b []byte) (int, error) { + n, err := d.r.Read(b) + d.h.Write(b[:n]) + if errors.Is(err, io.EOF) && "sha256:"+hex.EncodeToString(d.h.Sum(nil)) != d.want { + return n, errStoredMetadataChanged + } + return n, err +} + +func (d *digestReader) Close() error { return d.r.Close() } diff --git a/internal/handler/metadata_stream.go b/internal/handler/metadata_stream.go index 56606e2..c165b81 100644 --- a/internal/handler/metadata_stream.go +++ b/internal/handler/metadata_stream.go @@ -35,6 +35,12 @@ func (p *Proxy) serveStoredMetadata(w http.ResponseWriter, r *http.Request, ecos if entry == nil { return false } + return p.serveMetadataEntry(w, r, ecosystem, cacheKey, entry, immutable) +} + +// serveMetadataEntry is serveStoredMetadata for entry, a row +// storedMetadataEntry or streamableMetadata accepted. +func (p *Proxy) serveMetadataEntry(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, entry *database.MetadataCacheEntry, immutable bool) bool { body, err := p.openMetadataEntry(r.Context(), ecosystem, cacheKey, entry) if err != nil { return false @@ -88,14 +94,20 @@ func (p *Proxy) storedMetadataEntry(ecosystem, cacheKey string, immutable bool) if !immutable && time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { return nil } + if !p.streamableMetadata(entry) { + return nil + } + return entry +} + +// streamableMetadata reports whether entry records a sha256 digest and a +// size within the metadata size limit, so its document can be streamed. +func (p *Proxy) streamableMetadata(entry *database.MetadataCacheEntry) bool { limit := p.MetadataMaxSize if limit <= 0 { limit = defaultMetadataMaxSize } - if !entry.ContentDigest.Valid || !strings.HasPrefix(entry.ContentDigest.String, "sha256:") || !entry.Size.Valid || entry.Size.Int64 > limit { - return nil - } - return entry + return entry.ContentDigest.Valid && strings.HasPrefix(entry.ContentDigest.String, "sha256:") && entry.Size.Valid && entry.Size.Int64 <= limit } // openMetadataEntry opens the stored document of a row freshMetadataEntry diff --git a/internal/handler/metadata_truncated_test.go b/internal/handler/metadata_truncated_test.go new file mode 100644 index 0000000..5044374 --- /dev/null +++ b/internal/handler/metadata_truncated_test.go @@ -0,0 +1,121 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/cooldown" +) + +// cutMetadataUpstream serves one npm packument and, when cut is set, closes +// the connection halfway through the document it announced. +type cutMetadataUpstream struct { + *httptest.Server + mu sync.Mutex + body string + cut bool +} + +func newCutMetadataUpstream(t *testing.T, body string) *cutMetadataUpstream { + t.Helper() + u := &cutMetadataUpstream{body: body} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + body, cut := u.body, u.cut + u.mu.Unlock() + if r.URL.Path != "/pkg" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Length", fmt.Sprint(len(body))) + if cut { + _, _ = io.WriteString(w, body[:len(body)/2]) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *cutMetadataUpstream) set(body string, cut bool) { + u.mu.Lock() + defer u.mu.Unlock() + u.body, u.cut = body, cut +} + +// truncPackument is an npm packument with n versions published long ago. +func truncPackument(n int) string { + var versions, times []string + for i := range n { + v := fmt.Sprintf("1.0.%d", i) + versions = append(versions, fmt.Sprintf(`%q:{"name":"pkg","version":%q,"dist":{"tarball":"https://registry.npmjs.org/pkg/-/pkg-%s.tgz"}}`, v, v, v)) + times = append(times, fmt.Sprintf(`%q:"2020-01-01T00:00:00Z"`, v)) + } + return `{"name":"pkg","dist-tags":{"latest":"1.0.0"},"versions":{` + strings.Join(versions, ",") + + `},"time":{` + strings.Join(times, ",") + `}}` +} + +// TestMetadataTruncatedUpstream checks that a document upstream cuts off +// before its Content-Length is neither served nor cached: the cached copy is +// served instead, with and without cooldown. +func TestMetadataTruncatedUpstream(t *testing.T) { + for _, withCooldown := range []bool{false, true} { + t.Run(map[bool]string{false: "no cooldown", true: "cooldown"}[withCooldown], func(t *testing.T) { + u := newCutMetadataUpstream(t, truncPackument(50)) + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + proxy.CacheMetadata = true + if withCooldown { + proxy.Cooldown = &cooldown.Config{Default: "3d"} + } + handler := NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() + + first := serve(handler, "/pkg", nil) + if first.Code != http.StatusOK { + t.Fatalf("first request: status %d", first.Code) + } + before, err := db.GetMetadataCache("npm", "pkg") + if err != nil || before == nil { + t.Fatalf("no metadata row after the first request: %v", err) + } + + u.set(truncPackument(51), true) + expireMetadata(t, db, "npm", "pkg") + proxy.SetMetadataRewriteCacheSize(0) + + rec := serve(handler, "/pkg", nil) + if rec.Code != http.StatusOK || rec.Body.String() != first.Body.String() { + t.Errorf("status %d, want 200 with the cached document", rec.Code) + } + after, err := db.GetMetadataCache("npm", "pkg") + if err != nil || after == nil || after.ContentDigest != before.ContentDigest { + t.Error("the cut-off document replaced the cached copy") + } + }) + } +} + +// TestMetadataTruncatedUpstreamUncached checks that a cut-off document with +// nothing cached fails the request and is not cached. +func TestMetadataTruncatedUpstreamUncached(t *testing.T) { + u := newCutMetadataUpstream(t, truncPackument(50)) + u.set(truncPackument(50), true) + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + proxy.CacheMetadata = true + handler := NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() + + if rec := serve(handler, "/pkg", nil); rec.Code != http.StatusBadGateway { + t.Errorf("status %d, want 502", rec.Code) + } + if entry, _ := db.GetMetadataCache("npm", "pkg"); entry != nil { + t.Error("cached the cut-off document") + } +} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 62bf440..3cf3244 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -2,6 +2,7 @@ package handler import ( "bytes" + "context" "errors" "fmt" "io" @@ -12,9 +13,11 @@ import ( "time" "github.com/git-pkgs/integrity" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/jsonscan" "github.com/git-pkgs/proxy/internal/metadata/npm" "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/proxy/internal/storage" ) const ( @@ -46,7 +49,11 @@ const ( // npm tarballs are stored under pkg:npm/{name} as purl builds it, so package // rules use the default key mapping. func init() { - retention.Default.Register(retention.Spec{Key: "npm", CanonicalPackage: retention.DefaultCanonical("npm")}) + retention.Default.Register(retention.Spec{ + Key: "npm", + CanonicalPackage: retention.DefaultCanonical("npm"), + CompareVersions: retention.VersionOrder("npm"), + }) } // NPMHandler handles npm registry protocol requests. @@ -264,6 +271,10 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } + if !versionLookup && h.serveStreamedPackument(w, r, packageName, upstreamURL, accept, metadataCacheKey) { + return + } + var validate func([]byte) error if !versionLookup { validate = h.packumentValidator() @@ -317,6 +328,223 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques serveRewrittenMetadata(w, r, rewritten) } +// serveStreamedPackument serves a packument rewritten from storage: the +// document is streamed from upstream into storage, then read twice, once to +// collect the time map and version names cooldown and the denylist decide +// on, and once to copy each kept version. registry.npmjs.org writes +// dist-tags before versions and time after them, so one pass can't do it. +// Neither the packument nor its rewrite is held in memory uncompressed, and +// packuments larger than metadata_max_size can be served. +// +// It reports false when it leaves the request to the in-memory rewrite: +// without metadata or rewrite caching, for a document it can't stream, such +// as an abbreviated packument where the full one is required, and when the +// rewrite fails. The document is stored by then, so the in-memory path reads +// it from storage. +func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, accept, metadataCacheKey string) bool { + if h.proxy.rewrites == nil { + return false + } + fullRequired := h.packumentValidator() != nil + options := h.npmRewriteOptions(packageName, nil) + var rewritten rewrittenMetadata + err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, func(entry *database.MetadataCacheEntry) (err error) { + rewritten, err = h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { + var scan *npm.Scan + err := readStored(open, func(in io.Reader) (err error) { + scan, err = npm.ScanPackument(in) + return err + }) + if err != nil { + return err + } + if fullRequired && scan.Abbreviated() { + return npm.ErrNotStreamable + } + return readStored(open, func(in io.Reader) error { + _, err := npm.RewriteStream(packageName, scan, in, out, options) + return err + }) + }) + return err + }) + switch { + case errors.Is(fetchErr, errInMemory): + return false + case fetchErr != nil: + h.writeFetchError(w, fetchErr) + return true + } + if err != nil { + if r.Context().Err() != nil { + return true // the client left while waiting on a shared rewrite + } + if !errors.Is(err, npm.ErrNotStreamable) { + h.proxy.Logger.Warn("failed to rewrite stored npm metadata, rewriting in memory", + "package", packageName, "error", err) + } + return false + } + serveRewrittenMetadata(w, r, rewritten) + return true +} + +// npmFetchError is a failure to fetch a packument, answered as such rather +// than as a failure to filter it. +type npmFetchError struct{ err error } + +func (e *npmFetchError) Error() string { return e.err.Error() } +func (e *npmFetchError) Unwrap() error { return e.err } + +// writeFetchError answers a request whose packument could not be fetched. +func (h *NPMHandler) writeFetchError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, ErrUpstreamNotFound): + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) + case writeUpstreamHeld(w, err): + default: + h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) + JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + } +} + +// errInMemory is returned when a packument has to be read into memory to +// answer the request: metadata is not cached in storage, or the stored +// document can't be read as a stream. +var errInMemory = errors.New("npm packument needs the in-memory path") + +// storedPackument returns the lookup of the packument stored for cacheKey, +// fetching it into storage first when needed. Its err is errInMemory when +// metadata isn't cached in storage, and ErrUpstreamNotFound and upstream +// errors as they come. +// With refresh, a stored copy within its TTL is fetched again rather than +// served. +func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool) metadataResult { + var gone string + if refresh && h.proxy.DB != nil { + if row, _ := h.proxy.DB.GetMetadataCache("npm", cacheKey); usableStoredMetadata(row) { + gone = row.ContentDigest.String + } + } + res := h.proxy.storedMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), gone) + if errors.Is(res.err, errMetadataNotStored) { + res.err = errInMemory + } + return res +} + +// readStoredPackument reads the packument stored for cacheKey with read, +// fetching it into storage first when needed. When the stored copy is gone +// or no longer has the digest its row records, the packument is fetched +// again and read once more. fetchErr is what storedPackument returned when the packument +// couldn't be fetched, and err is what read returned. +func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { + fetched, err := h.lookupStoredPackument(ctx, packumentURL, accept, cacheKey, false, read) + return err, fetched.err +} + +// lookupStoredPackument implements readStoredPackument. fetched is the +// lookup of the packument read, whose err is set when it couldn't be +// fetched. +func (h *NPMHandler) lookupStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool, read func(*database.MetadataCacheEntry) error) (fetched metadataResult, err error) { + fetched = h.storedPackument(ctx, packumentURL, accept, cacheKey, refresh) + if fetched.err != nil { + return fetched, nil + } + if err = read(fetched.entry); !errors.Is(err, storage.ErrNotFound) && !errors.Is(err, errStoredMetadataChanged) { + return fetched, err + } + h.proxy.Logger.Warn("stored npm metadata is missing or changed, fetching it again", "url", packumentURL, "error", err) + fetched = h.proxy.storedMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), fetched.entry.ContentDigest.String) + if errors.Is(fetched.err, errMetadataNotStored) { + fetched.err = errInMemory + } + if fetched.err != nil { + return fetched, nil + } + return fetched, read(fetched.entry) +} + +// packumentStreamValidator is packumentValidator for a packument read from +// storage. +func (h *NPMHandler) packumentStreamValidator() func(io.Reader) error { + if h.packumentValidator() == nil { + return nil + } + return func(r io.Reader) error { + entry, err := npm.LookupVersion(r, "") + if err != nil { + return err + } + if entry.Abbreviated { + return errAbbreviatedPackument + } + return nil + } +} + +// streamedVersionMetadata resolves a version or dist-tag through the +// packument stored in storage, as rewriteNPMVersionFromPackument does in +// memory: one pass collects what cooldown and the denylist decide on, and a +// second finds the version entry. It returns errInMemory when the request +// has to take the in-memory path. +func (h *NPMHandler) streamedVersionMetadata(ctx context.Context, packageName, specifier, packumentURL, accept, cacheKey, metadataCacheKey string) (rewrittenMetadata, error) { + if h.proxy.rewrites == nil { + return rewrittenMetadata{}, errInMemory + } + fullRequired := h.packumentValidator() != nil + options := h.npmRewriteOptions(packageName, nil) + var m rewrittenMetadata + err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) (err error) { + m, err = h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { + var scan *npm.Scan + err := readStored(open, func(in io.Reader) (err error) { + scan, err = npm.ScanPackument(in) + return err + }) + if err != nil { + return err + } + if fullRequired && scan.Abbreviated() { + return npm.ErrNotStreamable + } + return readStored(open, func(in io.Reader) error { + version, err := npm.VersionStream(packageName, specifier, scan, in, options) + if err != nil { + return npmRewriteError(err) + } + _, err = out.Write(version) + return err + }) + }) + return err + }) + switch { + case errors.Is(fetchErr, errInMemory): + return rewrittenMetadata{}, fetchErr + case fetchErr != nil: + return rewrittenMetadata{}, &npmFetchError{fetchErr} + } + switch { + case err == nil, ctx.Err() != nil, errors.Is(err, ErrVersionDenied), errors.Is(err, ErrVersionNotFound): + return m, err + case !errors.Is(err, npm.ErrNotStreamable): + h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", + "package", packageName, "error", err) + } + return rewrittenMetadata{}, errInMemory +} + +// readStored opens a stored document and reads it with read. +func readStored(open func() (io.ReadCloser, error), read func(io.Reader) error) error { + in, err := open() + if err != nil { + return err + } + defer func() { _ = in.Close() }() + return read(in) +} + // packumentAccept returns the Accept header packuments are requested with. // It prefers the smaller abbreviated packument format but includes // application/json as a fallback so upstreams that reject the abbreviated type @@ -329,9 +557,10 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques // // Every request for a packument uses it, since they share one cache row: a // request with another Accept would replace the document the others are -// served with one in the other format. +// served with one in the other format. Cooldown profiles share that row too, +// so cooldown in any of them asks for full metadata in all of them. func (h *NPMHandler) packumentAccept() string { - if h.proxy.NPMFullMetadata || h.proxy.cooldownEnabled() { + if h.proxy.NPMFullMetadata || h.proxy.anyCooldownEnabled() { return contentTypeJSON } return npmAcceptDefault @@ -371,7 +600,19 @@ func requireFullPackument(body []byte) error { // filtered packument so cooldown and dist-tags match the root metadata response. func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey string) { _, packumentPath := npmMetadataPath(packageName) - packument, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, registryURL+"/"+packumentPath, "", h.packumentValidator(), accept) + packumentURL := registryURL + "/" + packumentPath + rewritten, err := h.streamedVersionMetadata(r.Context(), packageName, specifier, packumentURL, accept, cacheKey, metadataCacheKey) + var fetchErr *npmFetchError + switch { + case errors.As(err, &fetchErr): + h.writeFetchError(w, fetchErr.err) + return + case !errors.Is(err, errInMemory): + h.writeVersionMetadata(w, r, packageName, rewritten, err) + return + } + + packument, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) @@ -385,10 +626,16 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { + rewritten, err = h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { out, err := h.rewriteNPMVersionFromPackument(packageName, specifier, b) return out, nil, err }) + h.writeVersionMetadata(w, r, packageName, rewritten, err) +} + +// writeVersionMetadata answers a version metadata request with its rewrite +// or the error that kept it from being made. +func (h *NPMHandler) writeVersionMetadata(w http.ResponseWriter, r *http.Request, packageName string, rewritten rewrittenMetadata, err error) { if err != nil { if r.Context().Err() != nil { return @@ -549,10 +796,8 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } registryURL, cacheKey := h.sourceFor(packageName) - metadata := sync.OnceValues(func() ([]byte, error) { - upstreamURL := registryURL + "/" + url.PathEscape(packageName) - body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, "", h.packumentValidator(), h.packumentAccept()) - return body, err + metadata := sync.OnceValues(func() (npmVersionData, error) { + return h.versionData(r.Context(), packageName, registryURL+"/"+url.PathEscape(packageName), cacheKey, version) }) inCooldown, err := h.versionInCooldown(packageName, version, metadata) if err != nil { @@ -594,7 +839,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifactRequest(w, r, result) } -func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, version, filename string, metadata func() ([]byte, error)) (*CacheResult, error) { +func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, version, filename string, metadata func() (npmVersionData, error)) (*CacheResult, error) { cacheFilename := h.routes.cacheFilename(packageName, filename) if cached, err := h.proxy.GetCachedArtifact(r.Context(), "npm", packageName, version, cacheFilename); err != nil || cached != nil { return cached, err @@ -602,35 +847,35 @@ func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, versi // The packument says where the tarball is and what it must hash to. // Without it the tarball could not be verified, and an unverified copy // would be cached and served to everyone, so the download fails instead. - body, err := metadata() + data, err := metadata() if err != nil { return nil, fmt.Errorf("fetching npm metadata to verify tarball: %w", err) } downloadURL := fmt.Sprintf("%s/%s/-/%s", registryURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) - if tarball := npmVersionTarball(body, version); tarball != "" { + if tarball := npmEntryTarball(data.entry); tarball != "" { downloadURL, err = validateTarballURL(tarball, registryURL) if err != nil { return nil, err } } - sri := npmVersionIntegrity(body, version) + sri := npmEntryIntegrity(data.entry) return h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "npm", packageName, version, cacheFilename, downloadURL, sri) } -// npmVersionIntegrity returns the integrity a packument declares for a -// version's tarball: dist.integrity when it is a Subresource Integrity string +// npmEntryIntegrity returns the integrity a version's packument entry +// declares for its tarball: dist.integrity when it is a Subresource Integrity string // the proxy can verify, or else the SHA-1 in dist.shasum, which is all // versions published before integrity existed carry, as "sha1-". It // returns "" when neither is usable, and the tarball is fetched unverified as // before. -func npmVersionIntegrity(body []byte, version string) string { - sri, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "integrity") +func npmEntryIntegrity(entry []byte) string { + sri, _, _ := jsonscan.LookupString(entry, "dist", "integrity") if sri != "" { if _, err := integrity.ParseSRI(sri); err == nil { return sri } } - shasum, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "shasum") + shasum, _, _ := jsonscan.LookupString(entry, "dist", "shasum") return sha1Checksum(shasum) } @@ -638,10 +883,93 @@ func npmVersionIntegrity(body []byte, version string) string { // or "" if it has none. It reads that one value in place rather than decoding // every version, since it runs on each tarball the proxy has not cached yet. func npmVersionTarball(body []byte, version string) string { - tarball, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "tarball") + entry, _ := jsonscan.Lookup(body, "versions", version) + return npmEntryTarball(entry) +} + +// npmEntryTarball is npmVersionTarball for the version's entry. +func npmEntryTarball(entry []byte) string { + tarball, _, _ := jsonscan.LookupString(entry, "dist", "tarball") return tarball } +// npmVersionData is what a tarball download reads from the packument: the +// version's entry, nil when it has none, and its publish time from the time +// map, "" when it has none. +type npmVersionData struct { + entry []byte + published string +} + +// versionData reads what a tarball download needs about version from the +// packument: from storage in one pass when metadata is cached there, and +// otherwise from the packument read into memory. +// +// A packument cached within its TTL can predate the version. Under cooldown a +// missing entry would let the version through with no publish time, so the +// packument is fetched again and the version decided on what upstream lists +// now, as the Composer handler does. +func (h *NPMHandler) versionData(ctx context.Context, packageName, packumentURL, cacheKey, version string) (npmVersionData, error) { + data, fromCache, err := h.readVersionData(ctx, packumentURL, cacheKey, version, false) + if err == nil && data.entry == nil && fromCache && h.proxy.cooldownApplies("npm", packageName) { + h.proxy.Logger.Info("cooldown: cached npm metadata lacks version, fetching it again", + "package", packageName, "version", version) + data, _, err = h.readVersionData(ctx, packumentURL, cacheKey, version, true) + } + return data, err +} + +// readVersionData implements versionData. fromCache reports whether the +// packument was a cached copy, within its TTL or served stale; refresh +// fetches it from upstream. +func (h *NPMHandler) readVersionData(ctx context.Context, packumentURL, cacheKey, version string, refresh bool) (data npmVersionData, fromCache bool, err error) { + accept := h.packumentAccept() + var found npm.VersionEntry + fetched, err := h.lookupStoredPackument(ctx, packumentURL, accept, cacheKey, refresh, func(entry *database.MetadataCacheEntry) error { + return readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { + found, err = npm.LookupVersion(in, version) + return err + }) + }) + fetchErr := fetched.err + switch { + case fetchErr == nil && err == nil && (!found.Abbreviated || h.packumentValidator() == nil): + if refresh && fetched.stale { + return npmVersionData{}, false, errRefetchFailed(fetched) + } + return npmVersionData{entry: found.Entry, published: found.Published}, fetched.fromCache || fetched.stale, nil + case fetchErr == nil && err != nil: + h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) + case fetchErr != nil && !errors.Is(fetchErr, errInMemory): + return npmVersionData{}, false, fetchErr + } + + var res metadataResult + if refresh { + res = h.proxy.refreshMetadata(ctx, "npm", cacheKey, packumentURL, accept, "", h.packumentValidator()) + } else { + res = h.proxy.fetchMetadataDocument(ctx, "npm", cacheKey, packumentURL, accept, h.packumentValidator()) + } + if res.err != nil { + return npmVersionData{}, false, res.err + } + if refresh && res.stale { + return npmVersionData{}, false, errRefetchFailed(res) + } + entry, _ := jsonscan.Lookup(res.body, "versions", version) + published, _, err := jsonscan.LookupString(res.body, "time", version) + if err != nil { + return npmVersionData{}, false, fmt.Errorf("parsing npm metadata: %w", err) + } + return npmVersionData{entry: entry, published: published}, res.fromCache || res.stale, nil +} + +// errRefetchFailed is the error for a refetch that got only the stale copy +// back. That copy is what lacked the version, so it can't decide it. +func errRefetchFailed(res metadataResult) error { + return fmt.Errorf("npm metadata could not be fetched again: %w", res.upstreamErr) +} + func validateTarballURL(raw, registryURL string) (string, error) { return validateRegistryURL("tarball", raw, registryURL) } @@ -692,7 +1020,7 @@ func validateRegistryURL(what, raw, registryURL string) (string, error) { // for a private version of the same name, a private time would hold back the // public version, and moving a route to another upstream would keep the old // one's times. -func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() ([]byte, error)) (bool, error) { +func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() (npmVersionData, error)) (bool, error) { if !h.proxy.cooldownApplies("npm", packageName) { return false, nil } @@ -705,7 +1033,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun } } - body, err := metadata() + data, err := metadata() if errors.Is(err, ErrUpstreamNotFound) { return false, nil } @@ -713,11 +1041,8 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun return false, err } - published, ok, err := jsonscan.LookupString(body, "time", version) - if err != nil { - return false, fmt.Errorf("parsing npm metadata: %w", err) - } - if !ok { + published := data.published + if published == "" { return false, nil } diff --git a/internal/handler/npm_attestations.go b/internal/handler/npm_attestations.go index 2694af2..cc84003 100644 --- a/internal/handler/npm_attestations.go +++ b/internal/handler/npm_attestations.go @@ -40,10 +40,8 @@ func (h *NPMHandler) handleAttestations(w http.ResponseWriter, r *http.Request) } registryURL, cacheKey := h.sourceFor(packageName) - metadata := sync.OnceValues(func() ([]byte, error) { - upstreamURL := registryURL + "/" + url.PathEscape(packageName) - body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, "", h.packumentValidator(), h.packumentAccept()) - return body, err + metadata := sync.OnceValues(func() (npmVersionData, error) { + return h.versionData(r.Context(), packageName, registryURL+"/"+url.PathEscape(packageName), cacheKey, version) }) inCooldown, err := h.versionInCooldown(packageName, version, metadata) @@ -61,7 +59,7 @@ func (h *NPMHandler) handleAttestations(w http.ResponseWriter, r *http.Request) return } - packument, err := metadata() + data, err := metadata() if err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) @@ -74,7 +72,7 @@ func (h *NPMHandler) handleAttestations(w http.ResponseWriter, r *http.Request) JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") return } - listed, _, _ := jsonscan.LookupString(packument, "versions", version, "dist", "attestations", "url") + listed, _, _ := jsonscan.LookupString(data.entry, "dist", "attestations", "url") if listed == "" { JSONError(w, http.StatusNotFound, errNoAttestations.Error()) return diff --git a/internal/handler/npm_packument_accept_test.go b/internal/handler/npm_packument_accept_test.go index 1a87fdd..07f4e63 100644 --- a/internal/handler/npm_packument_accept_test.go +++ b/internal/handler/npm_packument_accept_test.go @@ -208,3 +208,53 @@ func TestNPMCooldownSkipsAbbreviatedStaleFallback(t *testing.T) { t.Errorf("served the abbreviated packument as a stale copy: %s", w.Body.String()) } } + +// Cooldown profiles share a packument's cache row with the unprefixed URLs. +// With cooldown on for only some of them, the others must not replace the +// full packument with the abbreviated one: the next request under cooldown +// would fetch it again, and have no stale copy when upstream fails. +func TestNPMCooldownProfilesShareFullPackument(t *testing.T) { + for _, tc := range []struct{ name, global, profile string }{ + {"cooldown in the profile", "0", "7d"}, + {"cooldown in the global policy", "7d", "0"}, + } { + t.Run(tc.name, func(t *testing.T) { + var fail atomic.Bool + var fullRequests atomic.Int64 + upstream := abbreviatedThenFullRegistry(t, &fail, &fullRequests) + proxy, db, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Nanosecond // every request revalidates + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = profilePolicy(t, tc.global) + global := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + profile := NewNPMHandler(proxy.WithCooldown(profilePolicy(t, tc.profile), 0), "http://proxy.test/cooldown/p", upstream.URL) + filtered, unfiltered := profile, global + if tc.global != "0" { + filtered, unfiltered = global, profile + } + + for _, h := range []*NPMHandler{filtered, unfiltered} { + if w := serveNPM(t, h, http.MethodGet, "/demo", nil, nil); w.Code != http.StatusOK { + t.Fatalf("metadata: status %d: %s", w.Code, w.Body.String()) + } + } + entry, err := db.GetMetadataCache("npm", "demo") + if err != nil || entry == nil { + t.Fatalf("cache row: %v, %v", entry, err) + } + if entry.ContentType.String != contentTypeJSON { + t.Errorf("cached packument Content-Type = %q, want %q", entry.ContentType.String, contentTypeJSON) + } + + fail.Store(true) + w := serveNPM(t, filtered, http.MethodGet, "/demo", nil, nil) + if w.Code != http.StatusOK { + t.Fatalf("metadata while upstream fails: status %d, want the stale full packument", w.Code) + } + if strings.Contains(w.Body.String(), "demo-2.0.0.tgz") { + t.Errorf("stale packument lists a version in cooldown: %s", w.Body.String()) + } + }) + } +} diff --git a/internal/handler/npm_rewrite_bench_test.go b/internal/handler/npm_rewrite_bench_test.go index 86552db..cf8b895 100644 --- a/internal/handler/npm_rewrite_bench_test.go +++ b/internal/handler/npm_rewrite_bench_test.go @@ -1,44 +1,13 @@ package handler import ( - "fmt" - "strings" "testing" + + "github.com/git-pkgs/proxy/internal/metadata/npm/npmbench" ) -// benchPackument builds a full packument shaped like a large real one: -// thousands of versions, each with dependencies, scripts and a readme. -func benchPackument(versions int) []byte { - var b strings.Builder - readme := strings.Repeat("Some documentation text. ", 160) - b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"1.0.` + fmt.Sprint(versions-1) + `"},"versions":{`) - for i := range versions { - if i > 0 { - b.WriteByte(',') - } - v := fmt.Sprintf("1.0.%d", i) - fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"description":"A big package","main":"index.js",`+ - `"scripts":{"test":"node test.js","build":"tsc -p ."},"dependencies":{`, v, v) - for d := range 25 { - if d > 0 { - b.WriteByte(',') - } - fmt.Fprintf(&b, `"dep-%d":"^%d.0.0"`, d, d) - } - fmt.Fprintf(&b, `},"readme":%q,"dist":{"shasum":"0123456789abcdef0123456789abcdef01234567",`+ - `"integrity":"sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",`+ - `"tarball":"https://registry.npmjs.org/big/-/big-%s.tgz"}}`, readme, v) - } - b.WriteString(`},"time":{"created":"2015-01-01T00:00:00.000Z"`) - for i := range versions { - fmt.Fprintf(&b, `,"1.0.%d":"2016-01-01T00:00:00.000Z"`, i) - } - b.WriteString(`}}`) - return []byte(b.String()) -} - func BenchmarkNPMRewriteMetadata(b *testing.B) { - body := benchPackument(5000) + body := npmbench.Packument(5000) h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy.example"} b.SetBytes(int64(len(body))) b.ReportAllocs() @@ -50,7 +19,7 @@ func BenchmarkNPMRewriteMetadata(b *testing.B) { } func BenchmarkNPMVersionTarball(b *testing.B) { - body := benchPackument(5000) + body := npmbench.Packument(5000) b.SetBytes(int64(len(body))) b.ReportAllocs() for b.Loop() { diff --git a/internal/handler/npm_stream_harden_test.go b/internal/handler/npm_stream_harden_test.go new file mode 100644 index 0000000..0cebf0a --- /dev/null +++ b/internal/handler/npm_stream_harden_test.go @@ -0,0 +1,238 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// bigMetadataFiles returns a copy of what is in storage under the metadata path +// of big, staged copies included. +func bigMetadataFiles(store *mockStorage) map[string]string { + store.mu.Lock() + defer store.mu.Unlock() + files := map[string]string{} + for path, b := range store.files { + if strings.HasPrefix(path, metadataStoragePath("npm", "big")) { + files[path] = string(b) + } + } + return files +} + +// TestNPMStreamedPackumentBlobCorrupt checks that a stored copy whose bytes +// no longer match the digest of its row, within its TTL, is fetched again +// rather than failing the request, also above metadata_max_size. +func TestNPMStreamedPackumentBlobCorrupt(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + decodeStreamed(t, serve(handler, "/big", nil)) + + path := metadataStoragePath("npm", "big") + store.mu.Lock() + store.files[path] = []byte(strings.Replace(body, "1.0.5", "1.0.X", 1)) + store.mu.Unlock() + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { + t.Errorf("served %d versions, want 400", len(doc.Versions)) + } + if got := bigMetadataFiles(store)[path]; got != body { + t.Error("the corrupt copy was not replaced") + } +} + +// TestNPMStreamedPackumentTruncated checks that a document upstream cuts +// off halfway neither replaces the stored copy, which is served instead, +// nor leaves anything behind, with and without cooldown. +func TestNPMStreamedPackumentTruncated(t *testing.T) { + for _, withCooldown := range []bool{true, false} { + t.Run(map[bool]string{true: "cooldown", false: "no cooldown"}[withCooldown], func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + if !withCooldown { + proxy.Cooldown = nil + } + proxy.MetadataMaxSize = int64(len(body) / 4) + want := len(decodeStreamed(t, serve(handler, "/big", nil)).Versions) + before := bigMetadataFiles(store) + + u.replace(bigPackument(401), `"v2"`) + u.truncate.Store(true) + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != want { + t.Errorf("served %d versions, want the stored %d", len(doc.Versions), want) + } + after := bigMetadataFiles(store) + if len(after) != len(before) { + t.Errorf("storage holds %d files, %d before", len(after), len(before)) + } + for path, b := range before { + if after[path] != b { + t.Errorf("%s was changed", path) + } + } + }) + } +} + +// TestNPMStreamedPackumentTruncatedUncached checks that a document upstream +// cuts off halfway fails the request and is not kept. +func TestNPMStreamedPackumentTruncatedUncached(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.Cooldown = nil + proxy.MetadataMaxSize = int64(len(body) / 4) + u.truncate.Store(true) + + if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { + t.Errorf("status %d, want 502", rec.Code) + } + if files := bigMetadataFiles(store); len(files) != 0 { + t.Errorf("left %d files in storage", len(files)) + } + if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { + t.Error("left a metadata cache row") + } +} + +// TestNPMStreamedPackumentUncachedErrors checks the answers to upstream +// failures with nothing cached. +func TestNPMStreamedPackumentUncachedErrors(t *testing.T) { + for _, tt := range []struct { + name string + set func(*streamUpstream) + want int + }{ + {"not found", func(u *streamUpstream) { u.gone.Store(true) }, http.StatusNotFound}, + {"upstream down", func(u *streamUpstream) { u.fail.Store(true) }, http.StatusBadGateway}, + } { + t.Run(tt.name, func(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + _, store, handler := streamProxy(t, u) + tt.set(u) + if rec := serve(handler, "/big", nil); rec.Code != tt.want { + t.Errorf("status %d, want %d", rec.Code, tt.want) + } + if files := bigMetadataFiles(store); len(files) != 0 { + t.Errorf("left %d files in storage", len(files)) + } + }) + } +} + +// TestNPMStreamedPackumentConcurrent checks that concurrent requests for an +// uncached packument share one upstream fetch and all get it. +func TestNPMStreamedPackumentConcurrent(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + const n = 16 + var wg sync.WaitGroup + recs := make([]*httptest.ResponseRecorder, n) + for i := range n { + wg.Go(func() { recs[i] = serve(handler, "/big", nil) }) + } + wg.Wait() + for _, rec := range recs { + if doc := decodeStreamed(t, rec); len(doc.Versions) != 400 { + t.Errorf("served %d versions, want 400", len(doc.Versions)) + } + } + if got := u.sent.Load(); got != 1 { + t.Errorf("upstream sent the document %d times, want once", got) + } +} + +// TestNPMStreamedPackumentClientRevalidates checks that a client sending +// the ETag it got gets a 304, and a new ETag once the document changes. +func TestNPMStreamedPackumentClientRevalidates(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + + first := serve(handler, "/big", nil) + etag := first.Header().Get("ETag") + if first.Code != http.StatusOK || etag == "" { + t.Fatalf("status %d, ETag %q", first.Code, etag) + } + if rec := serve(handler, "/big", http.Header{"If-None-Match": {etag}}); rec.Code != http.StatusNotModified { + t.Errorf("same document: status %d, want 304", rec.Code) + } + + u.replace(bigPackument(401), `"v2"`) + proxy.MetadataTTL = time.Nanosecond + rec := serve(handler, "/big", http.Header{"If-None-Match": {etag}}) + if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { + t.Errorf("changed document: status %d, ETag %q (was %q)", rec.Code, rec.Header().Get("ETag"), etag) + } + if doc := decodeStreamed(t, rec); len(doc.Versions) != 401 { + t.Errorf("served %d versions, want 401", len(doc.Versions)) + } +} + +// TestNPMStreamedPackumentOverStreamLimit checks that a document over +// MetadataStreamMaxSize is neither stored nor served: the stored copy is +// served instead, with and without cooldown, and with nothing stored the +// request fails. +func TestNPMStreamedPackumentOverStreamLimit(t *testing.T) { + for _, withCooldown := range []bool{true, false} { + t.Run(map[bool]string{true: "cooldown", false: "no cooldown"}[withCooldown], func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + if !withCooldown { + proxy.Cooldown = nil + } + proxy.MetadataMaxSize = int64(len(body) / 4) + proxy.MetadataStreamMaxSize = int64(len(body) + 1000) + want := len(decodeStreamed(t, serve(handler, "/big", nil)).Versions) + before := bigMetadataFiles(store) + + u.replace(bigPackument(500), `"v2"`) + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != want { + t.Errorf("served %d versions, want the stored %d", len(doc.Versions), want) + } + after := bigMetadataFiles(store) + if len(after) != len(before) { + t.Errorf("storage holds %d files, %d before", len(after), len(before)) + } + for path, b := range before { + if after[path] != b { + t.Errorf("%s was changed", path) + } + } + }) + } + t.Run("uncached", func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + proxy.MetadataStreamMaxSize = int64(len(body) / 2) + + if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { + t.Errorf("status %d, want 502", rec.Code) + } + if files := bigMetadataFiles(store); len(files) != 0 { + t.Errorf("left %d files in storage", len(files)) + } + if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { + t.Error("left a metadata cache row") + } + }) +} diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go new file mode 100644 index 0000000..5d2ab73 --- /dev/null +++ b/internal/handler/npm_stream_test.go @@ -0,0 +1,464 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +// bigPackument returns a packument laid out as registry.npmjs.org writes +// them, dist-tags before versions and time after, with n old versions padded +// so the document is large, and a latest published an hour ago. +func bigPackument(n int) string { + return bigPackumentAt("https://registry.npmjs.org", n) +} + +// bigPackumentAt is bigPackument with tarballs under base. +func bigPackumentAt(base string, n int) string { + var b strings.Builder + b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"9.0.0","old":"1.0.0"},"versions":{`) + times := []string{`"created":"2015-01-01T00:00:00Z"`} + for i := range n { + v := fmt.Sprintf("1.0.%d", i) + if i == 0 { + v = "1.0.0" + } + fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"readme":%q,"dist":{"tarball":"%s/big/-/big-%s.tgz"}},`, + v, v, strings.Repeat("x", 512), base, v) + times = append(times, fmt.Sprintf(`%q:"2016-01-01T00:%02d:%02dZ"`, v, i/60%60, i%60)) + } + recent := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) + b.WriteString(`"9.0.0":{"name":"big","version":"9.0.0","dist":{"tarball":"` + base + `/big/-/big-9.0.0.tgz"}}},`) + times = append(times, `"9.0.0":"`+recent+`"`) + b.WriteString(`"time":{` + strings.Join(times, ",") + `},"readme":"big"}`) + return b.String() +} + +type streamUpstream struct { + *httptest.Server + body atomic.Value // string + etag atomic.Value // string + fail atomic.Bool + gone atomic.Bool + // truncate makes upstream close the connection halfway through the + // document it announced. + truncate atomic.Bool + requests atomic.Int32 + // sent counts the responses that carried the document. + sent atomic.Int32 +} + +// replace makes upstream send body under a new ETag, so a revalidation +// gets it in a 200. +func (u *streamUpstream) replace(body, etag string) { + u.body.Store(body) + u.etag.Store(etag) +} + +func newStreamUpstream(t *testing.T, body string) *streamUpstream { + t.Helper() + u := &streamUpstream{} + u.replace(body, `"v1"`) + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.requests.Add(1) + switch { + case u.fail.Load(): + http.Error(w, "down", http.StatusBadGateway) + case u.gone.Load(): + http.NotFound(w, r) + case r.URL.Path == "/big" && r.Header.Get("If-None-Match") == u.etag.Load().(string): + w.WriteHeader(http.StatusNotModified) + case r.URL.Path == "/big": + u.sent.Add(1) + body := u.body.Load().(string) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("ETag", u.etag.Load().(string)) + if u.truncate.Load() { + w.Header().Set("Content-Length", fmt.Sprint(len(body))) + _, _ = io.WriteString(w, body[:len(body)/2]) + return + } + _, _ = io.WriteString(w, body) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.Close) + return u +} + +func streamProxy(t *testing.T, u *streamUpstream) (*Proxy, *mockStorage, http.Handler) { + t.Helper() + proxy, _, store, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(64 << 20) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + return proxy, store, NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() +} + +type packumentDoc struct { + DistTags map[string]string `json:"dist-tags"` + Versions map[string]json.RawMessage `json:"versions"` + Time map[string]string `json:"time"` +} + +func decodeStreamed(t *testing.T, rec *httptest.ResponseRecorder) packumentDoc { + t.Helper() + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + body := rec.Body.Bytes() + if rec.Header().Get("Content-Encoding") == "gzip" { + zr, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if body, err = io.ReadAll(zr); err != nil { + t.Fatal(err) + } + } + var doc packumentDoc + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatalf("response is not JSON: %v", err) + } + return doc +} + +// TestNPMStreamedPackument requests a packument larger than +// metadata_max_size with cooldown on, its time map after its versions and +// its latest inside cooldown. The in-memory rewrite can't read it at all. +func TestNPMStreamedPackument(t *testing.T) { + const versions = 400 + body := bigPackument(versions) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + + for _, header := range []http.Header{nil, {"Accept-Encoding": {"gzip"}}} { + doc := decodeStreamed(t, serve(handler, "/big", header)) + if len(doc.Versions) != versions { + t.Errorf("%d versions, want %d without 9.0.0", len(doc.Versions), versions) + } + if _, ok := doc.Versions["9.0.0"]; ok { + t.Error("9.0.0 is listed although it is inside cooldown") + } + if _, ok := doc.Time["9.0.0"]; ok { + t.Error("time still lists 9.0.0") + } + if doc.DistTags["latest"] != fmt.Sprintf("1.0.%d", versions-1) || doc.DistTags["old"] != "1.0.0" { + t.Errorf("dist-tags = %v, want latest moved to the newest version left", doc.DistTags) + } + if !strings.Contains(string(doc.Versions["1.0.0"]), `"tarball":"http://proxy.example/npm/big/-/big-1.0.0.tgz"`) { + t.Errorf("tarball not rewritten: %s", doc.Versions["1.0.0"]) + } + } + if n := u.requests.Load(); n != 1 { + t.Errorf("upstream asked %d times, want once", n) + } + + // Without the rewrite cache the in-memory path runs, which the size + // limit stops. + proxy.SetMetadataRewriteCacheSize(0) + if rec := serve(handler, "/big", nil); rec.Code == http.StatusOK { + t.Error("in-memory rewrite served a packument over metadata_max_size; the test no longer covers the stream") + } +} + +// TestNPMStreamedPackumentMatchesInMemory checks that both paths serve the +// same document and ETag. +func TestNPMStreamedPackumentMatchesInMemory(t *testing.T) { + body := bigPackument(50) + u := newStreamUpstream(t, body) + _, _, streamed := streamProxy(t, u) + inMemory, _, handler := streamProxy(t, u) + inMemory.CacheMetadata = false + + a, b := serve(streamed, "/big", nil), serve(handler, "/big", nil) + if a.Code != http.StatusOK || b.Code != http.StatusOK { + t.Fatalf("status %d and %d", a.Code, b.Code) + } + var ca, cb bytes.Buffer + if err := json.Compact(&ca, a.Body.Bytes()); err != nil { + t.Fatal(err) + } + if err := json.Compact(&cb, b.Body.Bytes()); err != nil { + t.Fatal(err) + } + if !bytes.Equal(ca.Bytes(), cb.Bytes()) { + t.Error("streamed and in-memory rewrites differ") + } +} + +// TestNPMStreamedPackumentStale serves the stored copy when upstream fails +// after the TTL, and 404 for a package upstream doesn't have. +func TestNPMStreamedPackumentStale(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, _, handler := streamProxy(t, u) + decodeStreamed(t, serve(handler, "/big", nil)) + + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) // drop the cached rewrite + u.fail.Store(true) + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { + t.Errorf("stale copy has %d versions, want 10", len(doc.Versions)) + } + + u.fail.Store(false) + if rec := serve(handler, "/missing", nil); rec.Code != http.StatusNotFound { + t.Errorf("missing package: status %d, want 404", rec.Code) + } +} + +// TestNPMStreamedPackumentStaleOn404 checks that a stored copy past its TTL +// is served when upstream answers 404, as the in-memory path does. +func TestNPMStreamedPackumentStaleOn404(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, _, handler := streamProxy(t, u) + decodeStreamed(t, serve(handler, "/big", nil)) + + proxy.MetadataTTL = time.Nanosecond + u.gone.Store(true) + for _, size := range []int64{64 << 20, 0} { // streamed, then in memory + proxy.SetMetadataRewriteCacheSize(size) + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { + t.Errorf("rewrite cache %d: stale copy has %d versions, want 10", size, len(doc.Versions)) + } + } +} + +// TestNPMStreamedPackumentBlobMissing checks that a row whose document is +// gone from storage, within its TTL or revalidated by a 304 after it, fetches +// the document again, also when it is larger than metadata_max_size. +func TestNPMStreamedPackumentBlobMissing(t *testing.T) { + for _, tt := range []struct { + name string + ttl time.Duration + }{ + {"fresh row", time.Hour}, + {"expired row", time.Nanosecond}, + } { + t.Run(tt.name, func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + decodeStreamed(t, serve(handler, "/big", nil)) + + path := metadataStoragePath("npm", "big") + store.mu.Lock() + delete(store.files, path) + store.mu.Unlock() + proxy.MetadataTTL = tt.ttl + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { + t.Errorf("served %d versions, want 400", len(doc.Versions)) + } + if n := u.sent.Load(); n != 2 { + t.Errorf("upstream sent the document %d times, want 2", n) + } + store.mu.Lock() + defer store.mu.Unlock() + if string(store.files[path]) != body { + t.Error("the document was not stored again") + } + }) + } +} + +// TestNPMStreamedVersionBlobMissing checks that a version request and a +// tarball download fetch a packument again whose stored copy is gone while +// its row is within the TTL. +func TestNPMStreamedVersionBlobMissing(t *testing.T) { + u := newStreamUpstream(t, "") + u.body.Store(bigPackumentAt(u.URL, 100)) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = 1024 + fetcher := proxy.Fetcher.(*mockFetcher) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} + path := metadataStoragePath("npm", "big") + + for _, req := range []string{"/big/1.0.5", "/big/-/big-1.0.5.tgz"} { + decodeStreamed(t, serve(handler, "/big", nil)) + store.mu.Lock() + delete(store.files, path) + store.mu.Unlock() + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if rec := serve(handler, req, nil); rec.Code != http.StatusOK { + t.Errorf("%s: status %d: %.200s", req, rec.Code, rec.Body.String()) + } + store.mu.Lock() + _, ok := store.files[path] + store.mu.Unlock() + if !ok { + t.Errorf("%s: the packument was not stored again", req) + } + } +} + +// TestNPMStreamedPackumentChangedInStorage checks that a document replaced +// in storage between the two passes isn't rewritten from mixed bytes. +func TestNPMStreamedPackumentChangedInStorage(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, store, _ := streamProxy(t, u) + entry, err := proxy.storedMetadataDocument(context.Background(), "npm", "big", u.URL+"/big", contentTypeJSON, nil) + if err != nil { + t.Fatal(err) + } + store.mu.Lock() + store.files[entry.StoragePath] = []byte(bigPackument(11)) + store.mu.Unlock() + + r, err := proxy.openStoredMetadata(context.Background(), entry) + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + if _, err := io.ReadAll(r); !errors.Is(err, errStoredMetadataChanged) { + t.Errorf("read error %v, want errStoredMetadataChanged", err) + } +} + +// TestNPMStreamedPackumentRevalidates checks that a 304 after the TTL keeps +// the stored document and its cached rewrite, without storing it again. +func TestNPMStreamedPackumentRevalidates(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, store, handler := streamProxy(t, u) + first := serve(handler, "/big", nil) + decodeStreamed(t, first) + store.mu.Lock() + stores := store.stores + store.mu.Unlock() + + proxy.MetadataTTL = time.Nanosecond + second := serve(handler, "/big", nil) + decodeStreamed(t, second) + if second.Header().Get("ETag") != first.Header().Get("ETag") { + t.Errorf("ETag changed from %s to %s", first.Header().Get("ETag"), second.Header().Get("ETag")) + } + if n := u.requests.Load(); n != 2 { + t.Errorf("upstream asked %d times, want 2", n) + } + store.mu.Lock() + defer store.mu.Unlock() + if store.stores != stores { + t.Errorf("the 304 stored the document again (%d stores, %d before)", store.stores, stores) + } +} + +// TestNPMStreamedPackumentKeepsFullCopy checks that an abbreviated packument +// from upstream, where cooldown needs the full one, doesn't replace the full +// copy in storage, which is served instead as when upstream fails. +func TestNPMStreamedPackumentKeepsFullCopy(t *testing.T) { + full := bigPackument(10) + u := newStreamUpstream(t, full) + proxy, store, handler := streamProxy(t, u) + decodeStreamed(t, serve(handler, "/big", nil)) + + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) + u.replace(`{"name":"big","modified":"2026-01-01T00:00:00Z","dist-tags":{"latest":"1.0.0"},"versions":{"1.0.0":{}}}`, `"v2"`) + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { + t.Errorf("served %d versions, want the 10 of the full copy", len(doc.Versions)) + } + if n := u.sent.Load(); n != 2 { + t.Fatalf("upstream sent a document %d times, want 2: the abbreviated one was not fetched", n) + } + + store.mu.Lock() + defer store.mu.Unlock() + if got := string(store.files[metadataStoragePath("npm", "big")]); got != full { + t.Errorf("stored packument was replaced: %.80s", got) + } + for path := range store.files { + if strings.Contains(path, ".staged-") { + t.Errorf("staged copy %s left in storage", path) + } + } +} + +// TestNPMStreamedPackumentStorageFails checks that a packument is still +// served, read into memory, when storing it fails. +func TestNPMStreamedPackumentStorageFails(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + _, store, handler := streamProxy(t, u) + store.storeErr = errors.New("no space left on device") + doc := decodeStreamed(t, serve(handler, "/big", nil)) + if len(doc.Versions) != 10 || doc.DistTags["latest"] != "1.0.9" { + t.Errorf("served %d versions, latest %q; want 10 and 1.0.9", len(doc.Versions), doc.DistTags["latest"]) + } +} + +// TestNPMStreamedVersionMetadata resolves versions and dist-tags of a +// packument larger than metadata_max_size. +func TestNPMStreamedVersionMetadata(t *testing.T) { + body := bigPackument(100) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + + for path, want := range map[string]string{ + "/big/1.0.5": "1.0.5", + "/big/latest": "1.0.99", // 9.0.0 is inside cooldown + "/big/old": "1.0.0", + } { + rec := serve(handler, path, nil) + var v struct { + Version string `json:"version"` + Dist struct { + Tarball string `json:"tarball"` + } `json:"dist"` + } + if rec.Code != http.StatusOK || json.Unmarshal(rec.Body.Bytes(), &v) != nil { + t.Errorf("%s: status %d: %.200s", path, rec.Code, rec.Body.String()) + continue + } + if v.Version != want || v.Dist.Tarball != "http://proxy.example/npm/big/-/big-"+want+".tgz" { + t.Errorf("%s = %s with tarball %s, want %s", path, v.Version, v.Dist.Tarball, want) + } + } + if rec := serve(handler, "/big/9.0.0", nil); rec.Code != http.StatusNotFound { + t.Errorf("version inside cooldown: status %d, want 404", rec.Code) + } + if n := u.requests.Load(); n != 1 { + t.Errorf("upstream asked %d times, want once", n) + } +} + +// TestNPMStreamedTarballCooldown checks the tarball download against a +// packument larger than metadata_max_size: the version's publish time and +// tarball URL are read from storage. +func TestNPMStreamedTarballCooldown(t *testing.T) { + u := newStreamUpstream(t, "") + u.body.Store(bigPackumentAt(u.URL, 100)) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = 1024 + fetcher := proxy.Fetcher.(*mockFetcher) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} + + if rec := serve(handler, "/big/-/big-9.0.0.tgz", nil); rec.Code != http.StatusNotFound { + t.Errorf("tarball inside cooldown: status %d, want 404", rec.Code) + } + if rec := serve(handler, "/big/-/big-1.0.5.tgz", nil); rec.Code != http.StatusOK || rec.Body.String() != "package" { + t.Errorf("tarball: status %d, body %q", rec.Code, rec.Body.String()) + } + if want := u.URL + "/big/-/big-1.0.5.tgz"; fetcher.fetchedURL != want { + t.Errorf("fetched %s, want %s from the packument", fetcher.fetchedURL, want) + } +} diff --git a/internal/handler/npm_tarball_test.go b/internal/handler/npm_tarball_test.go index fe1508d..6bca0f6 100644 --- a/internal/handler/npm_tarball_test.go +++ b/internal/handler/npm_tarball_test.go @@ -1,6 +1,7 @@ package handler import ( + "context" "encoding/json" "fmt" "io" @@ -13,6 +14,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" ) @@ -201,3 +203,224 @@ func TestNPMTarballRejectsUnsafeMetadataURL(t *testing.T) { }) } } + +// TestNPMTarballCooldownRefetchesStalePackument checks that a tarball for a +// version the cached packument does not list yet is decided on the packument +// upstream serves now, not let through for want of a publish time. +func TestNPMTarballCooldownRefetchesStalePackument(t *testing.T) { + for _, tc := range []struct { + name string + published time.Time + down bool + want int + }{ + {"recent", time.Now().Add(-time.Hour), false, http.StatusNotFound}, + // Without upstream the version can't be checked, so it isn't served. + {"upstream down", time.Now().Add(-time.Hour), true, http.StatusBadGateway}, + // The refetched entry's integrity is checked, so a tarball that + // doesn't match it is refused rather than served unverified. + {"old", time.Now().Add(-30 * 24 * time.Hour), false, http.StatusBadGateway}, + } { + t.Run(tc.name, func(t *testing.T) { + var released atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + old := time.Now().Add(-60 * 24 * time.Hour).Format(time.RFC3339) + if released.Load() && tc.down { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + if !released.Load() { + _, _ = fmt.Fprintf(w, `{"name":"widget","time":{"1.0.0":%q},"versions":{"1.0.0":{"dist":{}}}}`, old) + return + } + _, _ = fmt.Fprintf(w, `{"name":"widget","time":{"1.0.0":%q,"2.0.0":%q},"versions":{"1.0.0":{"dist":{}},"2.0.0":{"dist":{"integrity":%q}}}}`, + old, tc.published.Format(time.RFC3339), sha512SRI("package")) + })) + t.Cleanup(upstream.Close) + p, _, _, fetcher := setupTestProxy(t) + p.CacheMetadata = true + p.MetadataTTL = time.Hour + p.Cooldown = &cooldown.Config{Default: "7d"} + p.HTTPClient = upstream.Client() + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("tampered"))} + h := NewNPMHandler(p, "http://proxy.test", upstream.URL) + + metadata := httptest.NewRecorder() + h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) + if metadata.Code != http.StatusOK { + t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) + } + released.Store(true) + + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) + if response.Code != tc.want { + t.Fatalf("status = %d, want %d: %s", response.Code, tc.want, response.Body.String()) + } + }) + } +} + +// widgetPackument is a packument listing only widget 1.0.0, published long +// ago. +func widgetPackument() string { + old := time.Now().Add(-60 * 24 * time.Hour).Format(time.RFC3339) + return fmt.Sprintf(`{"name":"widget","time":{"1.0.0":%q},"versions":{"1.0.0":{"dist":{}}}}`, old) +} + +// cooldownNPMHandler returns an npm handler for upstream with cooldown on +// and a tarball fetcher that would serve anything asked of it. +func cooldownNPMHandler(t *testing.T, upstream *httptest.Server, ttl time.Duration) *NPMHandler { + t.Helper() + p, _, _, fetcher := setupTestProxy(t) + p.CacheMetadata = true + p.MetadataTTL = ttl + p.Cooldown = &cooldown.Config{Default: "7d"} + p.HTTPClient = upstream.Client() + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} + return NewNPMHandler(p, "http://proxy.test", upstream.URL) +} + +// TestNPMTarballCooldownStalePackument checks that a stale packument served +// because upstream failed doesn't let through a version it doesn't list, +// and that the upstream failure decides the response. +func TestNPMTarballCooldownStalePackument(t *testing.T) { + for _, tc := range []struct { + name string + status int + limited bool + want int + }{ + {"upstream down", http.StatusServiceUnavailable, false, http.StatusBadGateway}, + {"not found", http.StatusNotFound, false, http.StatusNotFound}, + {"rate limited", http.StatusTooManyRequests, true, http.StatusServiceUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + var down atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if down.Load() { + w.Header().Set("Retry-After", "60") + w.WriteHeader(tc.status) + return + } + _, _ = io.WriteString(w, widgetPackument()) + })) + t.Cleanup(upstream.Close) + // With no TTL the packument is read in memory and every read + // asks upstream. + h := cooldownNPMHandler(t, upstream, 0) + if tc.limited { + gate := registryclient.NewRateLimitGate(http.DefaultTransport) + h.proxy.HTTPClient = &http.Client{Transport: gate} + h.proxy.RateLimits = gate + } + + metadata := httptest.NewRecorder() + h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) + if metadata.Code != http.StatusOK { + t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) + } + down.Store(true) + + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) + if response.Code != tc.want { + t.Fatalf("status = %d, want %d: %s", response.Code, tc.want, response.Body.String()) + } + if tc.limited && response.Header().Get("Retry-After") == "" { + t.Error("no Retry-After on a rate-limited response") + } + }) + } +} + +// TestNPMTarballCooldownFetchesMissingVersionOnce checks that a packument +// just fetched from upstream isn't fetched again because it doesn't list the +// requested version. +func TestNPMTarballCooldownFetchesMissingVersionOnce(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + _, _ = io.WriteString(w, widgetPackument()) + })) + t.Cleanup(upstream.Close) + h := cooldownNPMHandler(t, upstream, time.Hour) + + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-99.0.0.tgz", nil)) + if n := requests.Load(); n != 1 { + t.Errorf("upstream packument requests = %d, want 1", n) + } +} + +// TestNPMTarballCooldownRefreshIgnoresSharedStale checks that a refetch of a +// stored packument lacking the version, joining another caller's fetch that +// falls back to the stored copy, doesn't take that copy as upstream's answer. +func TestNPMTarballCooldownRefreshIgnoresSharedStale(t *testing.T) { + var blocked atomic.Bool + arrived := make(chan struct{}, 1) + release := make(chan struct{}) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !blocked.Load() { + _, _ = io.WriteString(w, widgetPackument()) + return + } + arrived <- struct{}{} + select { + case <-release: + case <-r.Context().Done(): + } + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(upstream.Close) + h := cooldownNPMHandler(t, upstream, time.Hour) + p := h.proxy + + metadata := httptest.NewRecorder() + h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) + if metadata.Code != http.StatusOK { + t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) + } + registryURL, cacheKey := h.sourceFor("widget") + packumentURL := registryURL + "/widget" + accept := h.packumentAccept() + row, err := p.DB.GetMetadataCache("npm", cacheKey) + if err != nil || row == nil { + t.Fatalf("no stored packument row: %v", err) + } + setFetchedAt := func(at time.Time) { + t.Helper() + if ok, err := p.DB.RefreshMetadataCache("npm", cacheKey, row.ContentDigest.String, at); err != nil || !ok { + t.Fatalf("setting fetched_at: %t, %v", ok, err) + } + } + + // Another caller revalidates the expired copy, and upstream holds it. + setFetchedAt(time.Now().Add(-2 * time.Hour)) + blocked.Store(true) + other := make(chan error, 1) + go func() { + _, err := p.storedMetadataDocument(context.Background(), "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator()) + other <- err + }() + <-arrived + + // The tarball request reads the copy as within its TTL and refetches it, + // joining that fetch. + setFetchedAt(time.Now()) + response := httptest.NewRecorder() + done := make(chan struct{}) + go func() { + defer close(done) + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) + }() + waitForMetadataWaiters(t, p, metadataCoalesceKey("npm", cacheKey, packumentURL, accept, "", false)+"\x00stored", 1) + close(release) + <-done + if err := <-other; err != nil { + t.Fatalf("other caller: %v", err) + } + if response.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusBadGateway, response.Body.String()) + } +} diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index 20fda4e..9b9a6c3 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -561,9 +561,9 @@ func TestNPMDownloadCooldownDisabled(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) h := NewNPMHandler(proxy, "http://proxy.test", "") - if inCooldown, _ := h.versionInCooldown("leftpad", testVersion100, func() ([]byte, error) { + if inCooldown, _ := h.versionInCooldown("leftpad", testVersion100, func() (npmVersionData, error) { t.Fatal("cooldown must not request metadata when disabled") - return nil, nil + return npmVersionData{}, nil }); inCooldown { t.Error("versionInCooldown = true, want false when cooldown is not configured") } diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index b9345c5..714f219 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -1,13 +1,16 @@ package handler import ( + "bufio" "bytes" "compress/gzip" "context" "crypto/sha256" "encoding/json" + "encoding/json/jsontext" "errors" "fmt" + "io" "net/http" "net/url" "slices" @@ -36,36 +39,45 @@ func (h *NuGetHandler) nugetCooldownApplies(id string) bool { // Cache upstream documents, not filtered results, so policy changes and elapsed // time take effect even while metadata is fresh. Include the upstream in the key. func (h *NuGetHandler) nugetMetadata(ctx context.Context, path string) (map[string]any, error) { - target := h.upstreamURL + path - key := fmt.Sprintf("_cooldown/%x", sha256.Sum256([]byte(target))) var document map[string]any - validate := func(body []byte) error { - var err error - document, err = h.decodeNuGetMetadata(body) + err := h.readNuGetMetadata(ctx, path, func(body []byte) (err error) { + document, err = decodeNuGetDocument(body) return err - } - _, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "nuget", key, target, "", validate) + }) if err != nil { return nil, err } return document, nil } -func (h *NuGetHandler) decodeNuGetMetadata(body []byte) (map[string]any, error) { - // Normally the HTTP transport decodes gzip. Also support compressed cached - // bytes and clients with transparent decompression disabled, with the same - // metadata limit applied to the decompressed document. - if bytes.HasPrefix(body, []byte{0x1f, 0x8b}) { - reader, err := gzip.NewReader(bytes.NewReader(body)) - if err != nil { - return nil, err - } - defer func() { _ = reader.Close() }() - body, err = h.proxy.ReadMetadata(reader) - if err != nil { - return nil, err +// readNuGetMetadata fetches a document and passes it to decode, which keeps +// what its caller needs. decode can run more than once, and the document is +// the one it last accepted. It must not modify body. +func (h *NuGetHandler) readNuGetMetadata(ctx context.Context, path string, decode func(body []byte) error) error { + target := h.upstreamURL + path + key := fmt.Sprintf("_cooldown/%x", sha256.Sum256([]byte(target))) + validate := func(body []byte) error { + // Normally the HTTP transport decodes gzip. Also support compressed cached + // bytes and clients with transparent decompression disabled, with the same + // metadata limit applied to the decompressed document. + if bytes.HasPrefix(body, []byte{0x1f, 0x8b}) { + reader, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return err + } + defer func() { _ = reader.Close() }() + body, err = h.proxy.ReadMetadata(reader) + if err != nil { + return err + } } + return decode(body) } + _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "nuget", key, target, "", validate) + return err +} + +func decodeNuGetDocument(body []byte) (map[string]any, error) { var document map[string]any if err := json.Unmarshal(body, &document); err != nil { return nil, fmt.Errorf("parsing NuGet metadata: %w", err) @@ -76,13 +88,208 @@ func (h *NuGetHandler) decodeNuGetMetadata(body []byte) (map[string]any, error) return document, nil } +// nugetTimes holds the members of a registration index, page or leaf that +// decide which versions are in cooldown. Reading only them skips nearly all of +// a registration. +type nugetTimes struct { + // ID is the node's link. It is kept for the pages of an index only. + ID string + Version string + Published string + CatalogEntry nugetCatalogTimes + // Items is nil unless the node has an items array. + Items []nugetTimes +} + +// nugetCatalogTimes is the catalogEntry of a registration leaf. It stays unset +// unless the entry is an object: standalone leaves carry a link there. +type nugetCatalogTimes struct { + set bool + version string + published string +} + +// release returns the version and publish time of a leaf. A leaf embedded in a +// page carries them in catalogEntry, a standalone leaf at its root. +func (t *nugetTimes) release() (version, published string) { + if t.CatalogEntry.set { + return t.CatalogEntry.version, t.CatalogEntry.published + } + return t.Version, t.Published +} + +func (h *NuGetHandler) nugetTimesMetadata(ctx context.Context, path string) (*nugetTimes, error) { + document, _, err := h.nugetTimesDocument(ctx, path) + return document, err +} + +// nugetTimesDocument is nugetTimesMetadata that also returns the document +// itself, for a caller that goes on to copy it. +func (h *NuGetHandler) nugetTimesDocument(ctx context.Context, path string) (*nugetTimes, []byte, error) { + var document *nugetTimes + var body []byte + err := h.readNuGetMetadata(ctx, path, func(candidate []byte) error { + decoded, err := decodeNuGetTimes(candidate) + if err == nil { + document, body = decoded, candidate + } + return err + }) + if err != nil { + return nil, nil, err + } + return document, body, nil +} + +// decodeNuGetTimes reads a registration document the way writeNuGetRegistration +// copies it, so that what is decided here holds for what is written there: +// with the same decoder, members by their exact names, and a member of an +// unexpected type left empty. A node that has one of these members twice is +// refused, as which of the two counts would be the reader's choice. +func decodeNuGetTimes(body []byte) (*nugetTimes, error) { + dec := newNuGetDecoder(body) + if dec.PeekKind() == 'n' { + return nil, fmt.Errorf("empty NuGet metadata") + } + document := &nugetTimes{} + var scratch []byte + var err error + if dec.PeekKind() == '{' { + err = document.decode(dec, 0, &scratch) + } else if _, err = dec.ReadValue(); err == nil { + err = errors.New("not an object") + } + if err == nil { + // Nothing may follow the document. + if _, err = dec.ReadToken(); errors.Is(err, io.EOF) { + return document, nil + } else if err == nil { + err = errors.New("data after the document") + } + } + return nil, fmt.Errorf("parsing NuGet metadata: %w", err) +} + +// decode reads the node dec is at, which is depth levels below the document. +func (t *nugetTimes) decode(dec *jsontext.Decoder, depth int, scratch *[]byte) error { + if _, err := dec.ReadToken(); err != nil { + return err + } + seen := 0 + for dec.PeekKind() != '}' { + kind, err := readNuGetMember(dec, &seen, scratch) + if err != nil { + return err + } + switch value := dec.PeekKind(); { + case kind == nugetMemberItems && value == '[': + err = t.decodeItems(dec, depth, scratch) + case kind == nugetMemberCatalogEntry && value == '{': + err = t.CatalogEntry.decode(dec, scratch) + case kind == nugetMemberID && value == '"' && depth == 1: + t.ID, err = readNuGetString(dec, scratch) + case kind == nugetMemberVersion && value == '"': + t.Version, err = readNuGetString(dec, scratch) + case kind == nugetMemberPublished && value == '"': + t.Published, err = readNuGetString(dec, scratch) + default: + err = dec.SkipValue() + } + if err != nil { + return err + } + } + _, err := dec.ReadToken() + return err +} + +// decodeItems reads the items array dec is at. A child that is not an object +// is an empty node. +func (t *nugetTimes) decodeItems(dec *jsontext.Decoder, depth int, scratch *[]byte) error { + if _, err := dec.ReadToken(); err != nil { + return err + } + t.Items = []nugetTimes{} + for dec.PeekKind() != ']' { + t.Items = append(t.Items, nugetTimes{}) + var err error + if dec.PeekKind() == '{' { + err = t.Items[len(t.Items)-1].decode(dec, depth+1, scratch) + } else { + err = dec.SkipValue() + } + if err != nil { + return err + } + } + _, err := dec.ReadToken() + return err +} + +func (e *nugetCatalogTimes) decode(dec *jsontext.Decoder, scratch *[]byte) error { + if _, err := dec.ReadToken(); err != nil { + return err + } + e.set = true + seen := 0 + for dec.PeekKind() != '}' { + kind, err := readNuGetMember(dec, &seen, scratch) + if err != nil { + return err + } + switch value := dec.PeekKind(); { + case kind == nugetMemberVersion && value == '"': + e.version, err = readNuGetString(dec, scratch) + case kind == nugetMemberPublished && value == '"': + e.published, err = readNuGetString(dec, scratch) + default: + err = dec.SkipValue() + } + if err != nil { + return err + } + } + _, err := dec.ReadToken() + return err +} + +// nugetDecidingMembers are the members that decide what a node is and whether +// its release is in cooldown. +const nugetDecidingMembers = 1<