Watch
1
0
Fork
You've already forked pkg-proxy
1
mirror of https://github.com/git-pkgs/proxy.git synced 2026-11-11 08:39:17 -05:00
pkg-proxy/internal/config/config.go

1784 lines
62 KiB
Go
Raw Permalink Normal View History

2026-01-20 21:52:44 +00:00
// Package config provides configuration loading and validation for the proxy server.
//
// Configuration can be provided via:
// - Command line flags (highest priority)
// - Environment variables (PROXY_ prefix)
// - Configuration file (YAML or JSON)
//
// Storage Configuration:
//
// The proxy supports multiple storage backends via gocloud.dev/blob:
//
// Local filesystem (default):
//
// storage:
// url: "file:///var/cache/proxy"
//
// Amazon S3:
//
// storage:
// url: "s3://bucket-name"
//
// S3-compatible (MinIO, etc.):
//
// storage:
// url: "s3://bucket?endpoint=http://localhost:9000"
//
// Google Cloud Storage:
//
// storage:
// url: "gs://bucket-name"
//
// For S3, configure credentials via AWS environment variables:
//
// AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
//
// For GCS, authentication uses Application Default Credentials. This supports
// GKE Workload Identity, attached service accounts on GCE and Cloud Run, and
// local credentials created by `gcloud auth application-default login`.
// When direct_serve is enabled without a private key, the GCS backend uses the
// IAM Credentials signBlob API. The service account must hold
// roles/iam.serviceAccountTokenCreator on itself.
//
// Database Configuration:
//
// The proxy supports two database backends:
//
// SQLite (default):
2026-01-20 21:52:44 +00:00
//
// database:
// driver: "sqlite"
2026-01-20 21:52:44 +00:00
// path: "/var/lib/proxy/cache.db"
//
// PostgreSQL:
//
// database:
// driver: "postgres"
// url: "postgres://user:password@localhost:5432/proxy?sslmode=disable"
//
// See config.example.yaml in the repository root for a complete example.
2026-01-20 21:52:44 +00:00
package config
import (
"encoding/base64"
2026-01-20 21:52:44 +00:00
"encoding/json"
"fmt"
Route containerd ns mirror requests to configured registries (#405) * Route containerd ns mirror requests to configured registries containerd's hosts.toml mirrors append ?ns=<registry-host> to every request. The container handler ignored it, so a single _default mirror entry could not serve more than one registry. ns is a closed-world lookup key and is never dialed. Docker Hub aliases and the host of upstream.oci_default select the default route, hosts of upstream.oci entries select their named upstream. Unknown hosts return NAME_UNKNOWN so containerd falls back to its next host. Registry URLs with a path are not indexed, because ns names the registry at the root of a host. Host collisions log a warning and resolve deterministically. With ns the path is the verbatim upstream repository, the reserved upstream/ prefix is rejected, and repository prefix routes such as Homebrew's do not apply. Cache names match the unprefixed and upstream/{name}/ routes, so all routes share cached blobs, manifests and tag lists. ns is no longer forwarded on tag-list requests, and the pagination Link is rewritten per request instead of at store time so clients on different routes get links for their own route. Refs #303 * Document containerd ns mirroring Refs #303 * Version the tag-list cache identity for the raw-link format Tag-list rows now store the upstream Link verbatim and rewrite it per request. Rows written by earlier versions hold a Link already rewritten for one route. Under the unchanged key an older binary running next to this one (rolling update on shared Postgres, or a rollback) would serve the raw Link unrewritten, sending clients to the wrong route for the next page. A format marker in the key keeps the two apart at the cost of one cache miss per tag list after the upgrade. Adversarial review finding F1: tag-list cache rows hold raw upstream Links under unchanged keys, so an older binary serves them unrewritten. * Qualify the README claim about ns mirror coverage Registries whose URL has a path and hosts shared by two entries are not reachable through ns; point to the configuration guide for both rules. Adversarial review finding F3: README says ns covers every upstream.oci registry. * Redact registry URLs in ns index warnings Configured registry URLs may carry userinfo credentials, which the new startup warnings wrote to the log verbatim. Log them with the password masked, and say precisely what a path-prefixed default registry means: its host is not indexed, while the Docker Hub aliases still select it. Adversarial review finding F2: startup warnings print registry URLs unredacted and the default-registry message is misleading. * Keep non-default ports apart in the ns index Dropping 80 and 443 regardless of scheme turned https://host:80 and https://host into the same key, so a request for one of them could land on the other registry. Only the default port of the URL's scheme is optional now. A configured host is indexed with and without that port, because image references spell it either way; any other port has to match exactly, and the ns value is used as containerd sends it, apart from case and IPv6 brackets. Since a URL now yields two keys that can belong to different routes, the collision warning lists the owner per key. The handler test runs two registries on one host that differ only in the port. Tests can't bind 80 or 443, so a dialer maps those addresses to the fake servers. * Allow upstream/{name}/ together with ns when they agree containerd sends ns on every request to a mirror host, override_path or not. Per-registry mirrors pointing at /v2/upstream/{name} therefore arrive as upstream/{name}/...?ns=<registry>, and refusing every prefixed name under ns broke them, including the only way to mirror an upstream whose URL has a path. Such requests are now handled like the prefix route without ns, as long as ns names that upstream's own host under the same port rules as the index. Any other ns on a prefixed name is still NAME_UNKNOWN, so ns can't be used to create cache entries under another registry's name. * README: containerd config_path and how to check it A _default/hosts.toml does nothing while CRI has no hosts directory configured, so add the config.toml snippet for containerd 2.x and 1.x. To check the setup, look at containerd config dump and pull with crictl, then find the request in the proxy log; ctr --hosts-dir reads the directory on its own and would succeed even without config_path. Also mention that existing per-registry override_path entries keep working and that k3s generates the directory itself. * Let the prefix route take ns for aliases and mirrored registries With upstream.oci.hub pointing at registry-1.docker.io and a docker.io hosts.toml mirror on /v2/upstream/hub, containerd sends ns=docker.io, and the prefix check only accepted the upstream's own host. The same happens whenever the upstream is a mirror of the registry the nodes pull from, say an Artifactory remote for ghcr.io: ns carries ghcr.io, the upstream host is something else, and the pull got a 404. The prefix already decides where the content comes from and which cache entries are used, so ns can't redirect anything there. The check now only refuses an ns that contradicts the prefix, meaning a host the proxy knows that belongs to another route. The Docker Hub aliases count as one host, and a host the proxy doesn't know at all is accepted, since a client can only arrive at the prefix through its own mirror entry. * Take docker.io on every prefix route A Docker Hub mirror configured as a named upstream (mirror.gcr.io, an Artifactory remote) with a docker.io hosts.toml pointing at /v2/upstream/hub got a 404 again: docker.io is always in the index for the default route, so the check saw a known host of another route. That setup worked before this branch. Docker Hub repository names have exactly two path components, so docker.io/upstream/... can never be a real image, and accepting the aliases on every prefix route shadows nothing. While here: the collision warning claimed ns requests go to the other route, which is only true without the prefix, and the README said pulls for unknown registries always get a 404, which isn't the case for paths under upstream/{name}/. * Only take a foreign ns on a prefix route when it is configured A containerd _default mirror sends the same request for a pull of unconfigured.example/upstream/mirror/owner/app as a ghcr.io hosts.toml with override_path does for a mirror upstream. Accepting any host the proxy didn't know therefore let such a pull be answered with owner/app from the mirror upstream, a different image, instead of a 404 that lets containerd go to the registry the image actually names. The prefix route now accepts ns only for the upstream's own host, for Docker Hub, and for hosts listed in the new upstream.oci_mirrors setting. That setting is how an Artifactory remote of ghcr.io keeps working: upstream: oci_mirrors: ghcr: ["ghcr.io"] The listed hosts also select the upstream for unprefixed ns requests, which makes a mirror with a path in its URL reachable through the _default entry as well. Hosts from configured URLs still win over mirror entries; the proxy warns about such overlaps at startup. * Fix the docs sentence on which ns a prefix route takes configuration.md still said a prefix request with ns is only refused when ns names another route's host. Since the last commit it is the other way round: only the upstream's own host, its oci_mirrors entries and Docker Hub get through. * Note what a host accepted on a prefix route gives up Listing ghcr.io for an upstream (or the upstream's own host) still has the ambiguity from the wildcard case: ghcr.io/upstream/ghcr/owner/app lands on that upstream. That is the price of an explicit entry and fine in practice, but it should be written down. * Test that oci_mirrors entries stay with their own upstream Two gaps in the coverage of the last change: nothing failed if a host listed for one upstream was also accepted on another upstream's prefix, and nothing checked that the server actually hands upstream.oci_mirrors to the container handler. The handler test now has a second upstream that must refuse ghcr.io, and a small server test pulls through a mirror entry on both the prefix and the unprefixed route. * Log when a prefix route refuses an ns Setups with a ghcr.io hosts.toml pointing at a mirror upstream worked before this branch and now need an oci_mirrors entry. Without one, containerd gets a 404, quietly pulls from ghcr.io directly, and nobody notices the proxy is being skipped. The refusal is now logged as a warning that names the upstream and the ns and points at upstream.oci_mirrors. * Drop a check in validateOCIMirrors that could never fire user@ghcr.io parses with ghcr.io as its host, so the parsed.Host != host comparison already rejects it and the extra parsed.User test was dead. A test case for a host with credentials keeps that path covered. * Reject an oci_mirrors host that is listed for two upstreams With ghcr.io under both art and nexus, both prefix routes took ns=ghcr.io while unprefixed requests went to whichever name sorted first, with nothing but a startup warning. Config validation now refuses the second entry, comparing hosts without case and without the default :443. * Log each refused ns once per upstream and refuse overlong values The refusal warning fired on every request, and a node with a missing oci_mirrors entry asks for the manifest and each layer, so the same line piled up. Each upstream/ns pair is now logged once, for up to 1024 pairs; after that every refusal is logged again. ns comes straight from the client and can be as long as the request line allows, so remembering it as is would let anyone pin a lot of memory with a handful of requests. No registry host is longer than 259 characters (a 253-character name plus a port), so anything longer is refused right away, before it is logged or remembered. * Compare oci_mirrors hosts the way the handler does The duplicate check lowercased the host and trimmed :443, while the handler splits host and port, brackets IPv6 literals and drops an empty port. So ghcr.io and ghcr.io:, or [::1] and [::1]:, passed as different hosts in two upstreams and then collided in the handler. The check now builds the same key the handler uses. * Rewrap the oci_mirrors paragraphs in the configuration docs Two sentences had been dropped into the middle of wrapped lines, and the prefix-route paragraph named Docker Hub twice in a row. Same content, shorter lines, one mention. * Stop logging ns refusals once the set is full Past 1024 different upstream/ns pairs every refusal was logged again, so anyone could fill the set with made-up hosts and then flood the log at warning level. When the set is full the proxy now says so once and keeps quiet after that. A real misconfiguration shows up long before that, with the first pull. * Test the limit on remembered ns refusals Nothing covered what happens once 1024 pairs are remembered. The new test fills the set, then sends a known pair and two new ones: the proxy must log the limit notice exactly once, log none of the refusals and keep the set at its size.
2026-10-10 08:35:50 +02:00
"net"
"net/url"
2026-01-20 21:52:44 +00:00
"os"
Add package routes for private Composer and npm registries (#426) * Add Composer package routes for private repositories - Add upstream.composer_routes mapping package name patterns to repositories - Serve matching packages only from their route, with no fallback to the default repository - Qualify metadata and artifact cache keys with the route's upstream - Give routed versions an empty notification-url so installs are not reported to notify-batch - Document routes in configuration docs, example config and README * Add npm package routes for private registries - Add upstream.npm_routes mapping package name patterns to registries - Serve matching packages only from their route, with no fallback to upstream.npm - Resolve and validate tarball URLs against the route's registry - Qualify metadata and artifact cache keys with the route's upstream - Document npm routes and the audit endpoint limitation * Share routed artifact filename qualification between handlers * Match Composer routes without the ~dev suffix Packagist serves a package's dev versions from {name}~dev.json, and the handler matched routes against the name with ~dev still attached. An exact route for example/library therefore missed /p2/example/library~dev.json, and the private package's dev metadata came from the default repository. - Match routes against the package name without ~dev, keeping it in the upstream URL and the cache key so the two files stay apart - Add a regression test with an exact route * Keep routed npm publish times out of the shared versions row The download cooldown check stores publish times under the plain version PURL, which does not say which registry a version came from. A time read from the default registry was trusted for a private version of the same name, so a private release published an hour ago passed a seven-day cooldown when the public one was a month old. - Skip the stored row for routed packages, both reading and writing, and take the publish time from the route's metadata, which is cached under the route - This also stops a private time from holding back the public version, and a route's previous upstream from lending its times to the new one - Cover adding a route after a public download, the reverse order, and moving a route to another upstream
2026-10-11 19:06:18 +02:00
"path"
2026-01-20 21:52:44 +00:00
"path/filepath"
"regexp"
"sort"
2026-01-20 21:52:44 +00:00
"strconv"
"strings"
"time"
2026-01-20 21:52:44 +00:00
"github.com/git-pkgs/proxy/internal/denylist"
Add age-based retention for cached artifacts (#428) * Add age-based retention for cached artifacts Cached artifacts stayed forever unless storage.max_size pushed them out. storage.retention adds a default, per-ecosystem and per-package duration after which an artifact nobody downloaded gets evicted. Ecosystems opt in one at a time: a handler registers with retention.Default, and until it does, configuring that ecosystem (or one of its packages) fails validation, and the default doesn't touch it. This commit registers none, so nothing changes yet. Startup logs which ecosystems the default covers and which it doesn't. An artifact counts as expired when both its fetch time and its last access are older than the cutoff. Checking only the last access would evict a refetched artifact again right away, since clearing a record keeps its old access time and a cache miss doesn't record a hit. The sweep scans artifacts in id windows, so no single query holds the SQLite connection for long, and checks each row against its own rule. Expired records are cleared and their files go through pending_deletes instead of being deleted directly, because a buffered hit can make an artifact that's being served right now look expired. Buffered hits are flushed before each sweep, and one sweep clears at most 10,000 artifacts. Since a big first sweep can queue a lot, reclaim now keeps working through due entries for up to 30 seconds per tick instead of stopping after 100, and pending_deletes gets an index on queued_at. Also: - proxy_artifacts_evicted_total{reason, ecosystem} counts LRU and retention evictions. - gradle.build_cache.max_age accepts "7d", as its comment always said. Refs #306 * Document storage retention Describes the new storage.retention block, the order rules apply in, that the default only covers ecosystems that support retention yet, and what to expect from the first sweep over an old cache. Also lists the new eviction counter and notes that gradle max_age takes days. Refs #306 * Fix retention timing on Postgres and tighten the sweep On Postgres, fetched_at and last_accessed_at are TIMESTAMP columns without a zone. They hold the local time the proxy wrote, but lib/pq reads them back labelled UTC. East of UTC that makes every artifact look hours younger than it is, so the sweep evicted late: two hours in Berlin, nine in Tokyo. The candidates' times are now read back as local time, and the database tests run on both SQLite and Postgres, with one that sets the local zone to Asia/Tokyo. A few smaller things: - When a delete fails, reclaim stops for the rest of the minute instead of trying every queued path. Otherwise a storage outage turns into thirty seconds of failed deletes and warnings every minute. - If the buffered hits can't be written, the sweep skips that round. Running on stale access times could clear something that is being downloaded right now. - DefaultCanonical only accepts a package key when rebuilding it gives the same PURL. For alpine and deb it would otherwise produce keys that never match. - sweep_interval has to be at least a minute. - New tests cover the eviction counter and an artifact that gets a hit between the scan and the clear. - The docs no longer say nothing changes without retention, and they point out that the example's ecosystem and package rules only pass validation once those ecosystems support retention. Refs #306 * Reject qualifiers and subpaths in retention package keys A rule names a whole package. Qualifiers like ?type=pom were dropped on the way to the stored PURL and the rule quietly covered every file of the package. Now such a key fails validation. Refs #306 * Say when a package key doesn't match how packages are stored A package rule for an ecosystem that supports retention, written in a form its packages aren't stored under (pkg:npm/babel/core without the @), failed with "retention is not supported for npm packages". Once the ecosystem is enabled that is no longer true. Such a key now fails with "does not match how npm packages are stored" and points to the package rule forms in the docs. Keys for ecosystems without retention support keep the old message. Refs #306 * Note that re-running the mirror keeps packages cached A mirror run over a package that is still cached goes through the normal cache lookup and records a download, so the package's age starts over. The docs only said that mirrored packages age from the time they were mirrored. Refs #306 * Say when an ecosystem takes no package rules Some ecosystems will support retention without package rules, because nothing in their stored records names a package the way a PURL would (conan, helm). A package key for one of them used to fail with "does not match how conan packages are stored; see the package rule forms", which sends people looking for a form that doesn't exist. It now says package rules aren't supported there and points to the ecosystem rule. Refs #306 * Use helm as the example of an ecosystem without package rules Conan gets package rules once its handler stores archives under the recipe name, so the config test that shows the refusal now uses helm, whose cache records name the repository, not the chart. Refs #306 * Say that each cached file ages on its own The retention docs said each artifact, each version of a package, ages on its own. A version can have several cached files (a jar and its pom, several wheels, Conan sources and binaries), and each of them ages separately. Refs #306 * Clear an expired artifact and queue its file in one transaction The sweep cleared the record and queued the file for deletion as two separate writes. If queueing failed, or the process stopped in between, the file stayed in storage with no record pointing at it and no queue entry, and no later sweep would find it again. Both writes now share a transaction, so a failed queue rolls the clear back and the next sweep retries. Refs #306 * Show artifact evictions on the analytics page proxy_artifacts_evicted_total had no place on /ui/analytics. The Runtime card now lists evicted artifacts by reason and ecosystem, without the failure tint, and the coverage test records an eviction so it doesn't depend on test order. Refs #306 * Store artifact times in UTC fetched_at and last_accessed_at are now written in UTC, and the retention cutoffs are compared in UTC, the same convention #434 uses for the metadata cache. lib/pq sends a time with its offset and a Postgres TIMESTAMP keeps only the wall clock, so UTC on write is what reads back correctly. That replaces reinterpreting the times as local on read. Rows written before this still hold local time. East of UTC they look younger by the offset until they are rewritten, so retention evicts them a few hours late once; west of UTC that much early. Refs #306 --------- Co-authored-by: t <t@t>
2026-10-12 10:06:01 +02:00
"github.com/git-pkgs/proxy/internal/retention"
"github.com/git-pkgs/purl"
2026-01-20 21:52:44 +00:00
"gopkg.in/yaml.v3"
)
// DefaultSwiftUpstream is the Swift Package Registry used when none is configured.
const DefaultSwiftUpstream = "https://tuist.dev/api/registry/swift"
2026-01-20 21:52:44 +00:00
// Config holds all configuration for the proxy server.
type Config struct {
// Listen is the address to listen on (e.g., ":8080", "127.0.0.1:8080").
Listen string `json:"listen" yaml:"listen"`
// BaseURL is the public URL where package endpoints are reachable.
// Used for rewriting package metadata URLs and shown to humans on the
// install guide so they know what to point their package manager at.
2026-01-20 21:52:44 +00:00
// Example: "https://proxy.example.com" or "http://localhost:8080"
BaseURL string `json:"base_url" yaml:"base_url"`
// UIBaseURL is the public URL where the web UI is reachable. Defaults to
// BaseURL when unset. Set this separately when the UI is served on a
// different hostname than the package endpoints — for example, the UI on a
// public domain behind auth while build machines hit a Docker network alias
// for the package endpoints.
// Example: "https://proxy.example.com/ui"
UIBaseURL string `json:"ui_base_url" yaml:"ui_base_url"`
// TrustForwardedFor attributes requests to the leftmost X-Forwarded-For
// entry instead of the TCP peer address, in the structured log, the access
// log and the request-source table on /ui/analytics.
//
// Enable this only when the proxy sits behind a load balancer or ingress
// that sets the header, because any client can send it: behind one it is
// the only way to see past the hop, in front of one it lets a caller choose
// what address it is attributed to.
TrustForwardedFor bool `json:"trust_forwarded_for" yaml:"trust_forwarded_for"`
// UIRequestSources shows the request-source table on /ui/analytics, which
// reports caller addresses, the tool each ran and how much each pulled.
//
// Off by default. The proxy has no authentication of its own, so until this
// is enabled /ui exposes what is cached rather than who called.
UIRequestSources bool `json:"ui_request_sources" yaml:"ui_request_sources"`
// UIFailureDetails lists the latest failures behind each failure counter on
// /ui/analytics: the package, its upstream URL or storage path, and the
// error. URLs are shown without credentials or query strings.
//
// Off by default, for the same reason as UIRequestSources: package names and
// upstream hosts say what the proxy's users depend on.
UIFailureDetails bool `json:"ui_failure_details" yaml:"ui_failure_details"`
2026-01-20 21:52:44 +00:00
// Storage configures artifact storage.
Storage StorageConfig `json:"storage" yaml:"storage"`
// Database configures the cache database.
Database DatabaseConfig `json:"database" yaml:"database"`
// Log configures logging.
Log LogConfig `json:"log" yaml:"log"`
// AccessLog configures the JSONL activity log.
AccessLog AccessLogConfig `json:"access_log" yaml:"access_log"`
2026-01-20 21:52:44 +00:00
// Upstream configures upstream registry URLs (optional overrides).
Upstream UpstreamConfig `json:"upstream" yaml:"upstream"`
// Cooldown configures version age filtering to mitigate supply chain attacks.
Cooldown CooldownConfig `json:"cooldown" yaml:"cooldown"`
// Denylist blocks explicitly configured package versions.
Denylist DenylistConfig `json:"denylist" yaml:"denylist"`
Add pre-cache artifact scanning hook (trivy/ClamAV/Wiz/custom) (#298) Runs fetched artifacts through pluggable external scanners after they're staged in storage but before they're committed to the cache DB, so a block verdict deletes the object instead of ever exposing it to a client. Scanners pull the staged bytes themselves via a short-lived HMAC-signed internal route rather than the proxy pushing bytes to them, keeping the mechanism storage-backend-agnostic and avoiding uploading potentially huge artifacts through the proxy's own egress. Hardening baked in from the start: the internal scan-fetch route is gated both at router-mount time and in the handler so it's inert whenever scanning is disabled or unsigned; the signing key is mandatory whenever scanning is enabled, enforced directly in scanner.NewGroup rather than relying on callers to invoke config validation; the scan call and the delete-on-block cleanup both run on a context detached from the client's, so a client disconnecting mid-scan can't be mistaken for a scanner failure, doesn't cause a legitimate artifact to be deleted, and doesn't leave a genuinely blocked artifact's bytes orphaned in storage; and scanner infrastructure errors (connection failures, internal hostnames) are never forwarded verbatim to anonymous clients, only a generic message. The scan-error metric also correctly distinguishes a scanner's own timeout from being cancelled because a sibling scanner already decided the verdict. Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-03 05:06:44 -04:00
// Scanning configures pre-cache artifact scanning (trivy, ClamAV, Wiz,
// or a custom service) to mitigate supply chain attacks.
Scanning ScanningConfig `json:"scanning" yaml:"scanning"`
// CacheMetadata enables caching of upstream metadata responses for offline fallback.
// When enabled, metadata is stored in the database and storage backend.
// The mirror command always enables this regardless of this setting.
CacheMetadata bool `json:"cache_metadata" yaml:"cache_metadata"`
// MetadataTTL is how long cached metadata is considered fresh before
// revalidating with upstream. Uses Go duration syntax (e.g. "5m", "1h").
// Default: "5m". Set to "0" to always revalidate.
MetadataTTL string `json:"metadata_ttl" yaml:"metadata_ttl"`
// MetadataMaxSize is the maximum size of an upstream metadata response
// the proxy will buffer (e.g. "100MB", "250MB"). Responses over this
// size return ErrMetadataTooLarge. Default: "100MB".
MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"`
// ArtifactMaxSize is the largest upstream artifact the proxy will fetch
// (e.g. "2GB"). Larger artifacts are refused and not cached. Default: ""
// (unlimited). "0" is also unlimited.
ArtifactMaxSize string `json:"artifact_max_size" yaml:"artifact_max_size"`
Cache rewritten npm and Composer metadata (#402) The npm and Composer handlers rewrite every metadata document they serve so that download URLs point at the proxy: decode the whole document into generic maps, change the URLs, encode it again, and for Composer expand the minified format first. That ran on every request, cached metadata included. With metadata caching on and upstream out of the picture, a cached request still cost 7 ms for @babel/core, 17 ms for symfony/console and 155 ms and 134 MB of allocations for typescript, and an 8 vCPU VM serving cached npm packuments to 20 clients ran the proxy at 450% CPU for about 1,100 requests a second. Rewritten documents are now kept in memory, keyed by the ecosystem, proxy URL, package and a SHA-256 of the raw document, so new bytes from upstream are rewritten again and nothing is served stale. Requests that arrive while a document is being rewritten wait for that rewrite rather than running their own; a waiter leaves when its client does, and the rewrite still completes and is cached. The denylist is fixed at startup, so it needs no place in the key. Cooldown filtering depends on the current time, so with cooldown on the cache is bypassed. metadata_rewrite_cache_size bounds the cache (default "256MB", least recently used out first, "0" to rewrite on every request). NewProxy callers keep the old behaviour unless they set it. Whole cached requests, measured locally: @babel/core 7.1 ms -> 0.38 ms, 54,489 -> 119 allocations typescript 155 ms -> 10.5 ms, 1.37M -> 133 allocations symfony/console 17.4 ms -> 0.48 ms, 165,909 -> 123 allocations What remains is reading the raw document from storage and hashing it.
2026-10-03 04:09:15 -07:00
// MetadataRewriteCacheSize is how much rewritten npm and Composer
// metadata to keep in memory, so a document is rewritten once rather than
// on every request (e.g. "256MB", "1GB"). Default: "256MB". Set to "0" to
// rewrite on every request.
MetadataRewriteCacheSize string `json:"metadata_rewrite_cache_size" yaml:"metadata_rewrite_cache_size"`
// HTTPTimeout is the timeout for individual upstream HTTP requests made
// by protocol handlers (metadata fetches, pass-through file requests).
// Uses Go duration syntax (e.g. "30s", "2m"). Default: "30s".
fix(server): tune the shared upstream transport defaults (#351) * fix(server): tune the shared upstream transport defaults server.serve builds the shared client with safehttp.New, which clones Go's default transport: MaxIdleConnsPerHost stays 0 (an effective limit of two idle connections per host) and ResponseHeaderTimeout stays 0. Handing that client to fetch.NewFetcher via fetch.WithHTTPClient replaces the fetcher's own defaults of 10 idle connections per host and a 60-second response-header timeout. Set both on the shared transport, matching the fetcher defaults: a second burst of concurrent cache misses to one registry now reuses its connections instead of re-dialling most of them, and an upstream that accepts a request but stalls before sending headers is cut off after 60 seconds rather than only by the client's overall timeout. Tests measure connection reuse across two concurrent bursts against a TLS upstream that counts accepted connections (Go default: at most two reused; tuned: all eight) and assert that a stall before headers fails with the response-header timeout. Fixes #327 * fix(server): reconcile http_timeout docs and tighten the transport tests The http_timeout documentation and the config comment said "0" disables the upstream timeout entirely. With a fixed 60-second ResponseHeaderTimeout on the shared transport that is no longer the whole story, so both now say that waiting for response headers stays bounded independently of the setting. Test cleanup from review: the 50ms settle between bursts was dead time (the transport returns a connection to the idle pool before the body's final Read returns, so burst returning already means the pool is settled); the maxNewInBurst sentinel became explicit min/max bounds per case; the stall test dropped the client.Timeout override and the elapsed-time assertion, which was redundant with the error-text check in any realistic run and whose failure message misattributed the cause, and its comment now says plainly that the field assertions pin production while the behavioural half runs at a lowered timeout. * test(server): hold each burst at the upstream instead of sleeping The reuse test kept a burst in flight with a 100ms handler sleep, so a process stall longer than that between spawning the goroutines and their dials let a request finish early and hand its connection to a sibling. Review reproduced this with forced stalls: the default-transport case then dialled 5 instead of 6 connections. The handler now answers only once burstSize requests are waiting at the same time. With HTTP/1.1 pinned that forces every burst onto burstSize distinct connections regardless of scheduling, and the measured counts stay exactly 6 new for Go's default and 0 for the tuned transport, also under the same forced stalls.
2026-09-16 09:11:06 +02:00
// Set to "0" to disable the timeout entirely. Independently of this
// setting, the shared transport gives up on an upstream that has not sent
// response headers within 60 seconds.
HTTPTimeout string `json:"http_timeout" yaml:"http_timeout"`
// MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP.
// Disabled by default to prevent unauthenticated users from triggering downloads.
MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"`
// Hosted configures pushing packages to the proxy (/api/hosted).
Hosted HostedConfig `json:"hosted" yaml:"hosted"`
// Gradle configures Gradle HttpBuildCache behavior.
Gradle GradleConfig `json:"gradle" yaml:"gradle"`
Add storage backend probe to /health (closes #73) (#119) * config: add Health.StorageProbeInterval * metrics: add proxy_health_probe_failures_total counter * server: add storageProbe with happy-path test * server: add storageProbe failure-mode tests * server: add healthCache with TTL, single-flight, transition logging * server: wire storage probe into /health * server: update TestHealthEndpoint for JSON; wire healthCache into newTestServer Also fix Windows file-locking issue in storageProbe: close the reader explicitly before Delete so the file handle is released prior to os.Remove. * server: clean up stale comment in storageProbe * docs: document storage health probe and new metric * docs: regenerate Swagger for /health JSON response * server: simplify rc.Close error handling in storageProbe * server: defer probe cleanup so size/open/read/verify failures don't leak objects Previously, storageProbe only called Delete on the success path. Any failure between Store and the final Delete (size mismatch, Open error, mid-stream read failure, content mismatch) left the probe object orphaned in the storage backend. With caching disabled and Kubernetes-rate probing, the leak could accumulate noticeably on backends like S3. Use a named return + defer to attempt Delete after every successful Store. The earlier-step failure remains the primary error; Delete failure only surfaces as step="delete" when nothing else went wrong. Add a table-driven test that asserts cleanup runs for each non-delete failure path. Reported by Copilot on #119. * config: validate health.storage_probe_interval in Config.Validate The new duration field was only validated at use time in newHealthCache. The existing codebase already validates other duration fields (MetadataTTL, DirectServeTTL, Gradle.MaxAge, Gradle.SweepInterval) in Config.Validate() so misconfiguration fails fast at startup with a config-key-specific error. Match that pattern. The parse-at-use code in newHealthCache stays as a safety net, mirroring the MetadataTTL precedent. Reported by Copilot on #119. * docs: lowercase "counter" in metrics table for consistency Other rows in the table use lowercase type names (counter/gauge/histogram). Match that style. Reported by Copilot on #119. * docs: include size-check step in /health probe description The probe is write → size-check → read → verify → delete; the architecture note was missing the size-check step. Reported by Copilot on #119. * server: address andrew's review on #119 - Drop unused callerCtx parameter from healthCache.Check (Check is now parameter-less; the comment-only "accepted for symmetry" justification wasn't carrying its weight). - Emit "storage": {"status": "skipped"} on DB short-circuit instead of omitting the key, so monitors expecting a fixed key set keep working. - Reject negative storage_probe_interval at config validation time (previously parsed and silently behaved like "0"). - Extract HealthConfig.Validate to keep Config.Validate under the gocognit threshold and match the existing GradleBuildCacheConfig pattern. - README Health Check section: note that /health is intended as a readiness probe rather than a liveness probe (Check holds a mutex for up to the 10s probe timeout). - cmd/proxy/main.go godoc: column-align the new env var with the surrounding Gradle entries. Reported by andrew on #119.
2026-05-22 14:14:01 +03:00
// Health configures the /health endpoint behavior.
Health HealthConfig `json:"health" yaml:"health"`
}
// CooldownConfig configures version cooldown periods.
// Versions published more recently than the cooldown are hidden from metadata responses.
type CooldownConfig struct {
// Default is the global default cooldown (e.g., "3d", "48h", "0" to disable).
Default string `json:"default" yaml:"default"`
// Ecosystems overrides the default for specific ecosystems.
Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"`
// Packages overrides the cooldown for specific packages (keyed by PURL).
// Valid PURL keys are normalized to canonical form before use.
Packages map[string]string `json:"packages" yaml:"packages"`
// PackagePatterns overrides the cooldown for packages whose PURLs match a glob.
// Exact package overrides take precedence over matching patterns.
PackagePatterns map[string]string `json:"package_patterns" yaml:"package_patterns"`
2026-01-20 21:52:44 +00:00
}
// NormalizedPackages returns a copy of the package overrides with valid PURL
// keys in canonical form. An explicitly canonical key wins over an equivalent
// noncanonical key, and invalid keys are preserved unchanged.
func (c *CooldownConfig) NormalizedPackages() map[string]string {
if c == nil || c.Packages == nil {
return nil
}
keys := make([]string, 0, len(c.Packages))
for key := range c.Packages {
keys = append(keys, key)
}
sort.Strings(keys)
normalized := make(map[string]string, len(c.Packages))
for _, key := range keys {
canonical := key
if parsed, err := purl.Parse(key); err == nil {
canonical = parsed.String()
}
if _, exists := normalized[canonical]; exists && key != canonical {
continue
}
normalized[canonical] = c.Packages[key]
}
return normalized
}
Add pre-cache artifact scanning hook (trivy/ClamAV/Wiz/custom) (#298) Runs fetched artifacts through pluggable external scanners after they're staged in storage but before they're committed to the cache DB, so a block verdict deletes the object instead of ever exposing it to a client. Scanners pull the staged bytes themselves via a short-lived HMAC-signed internal route rather than the proxy pushing bytes to them, keeping the mechanism storage-backend-agnostic and avoiding uploading potentially huge artifacts through the proxy's own egress. Hardening baked in from the start: the internal scan-fetch route is gated both at router-mount time and in the handler so it's inert whenever scanning is disabled or unsigned; the signing key is mandatory whenever scanning is enabled, enforced directly in scanner.NewGroup rather than relying on callers to invoke config validation; the scan call and the delete-on-block cleanup both run on a context detached from the client's, so a client disconnecting mid-scan can't be mistaken for a scanner failure, doesn't cause a legitimate artifact to be deleted, and doesn't leave a genuinely blocked artifact's bytes orphaned in storage; and scanner infrastructure errors (connection failures, internal hostnames) are never forwarded verbatim to anonymous clients, only a generic message. The scan-error metric also correctly distinguishes a scanner's own timeout from being cancelled because a sibling scanner already decided the verdict. Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-03 05:06:44 -04:00
// ScanningConfig configures pre-cache artifact scanning (e.g. trivy,
// ClamAV, Wiz, or a custom service) to mitigate supply chain attacks.
// Unlike Cooldown, which only looks at a version's publish timestamp,
// scanning inspects the actual artifact bytes before they become
// servable from cache.
type ScanningConfig struct {
// Enabled turns on the scan gate. When false (default), artifacts are
// cached exactly as if scanning didn't exist.
Enabled bool `json:"enabled" yaml:"enabled"`
// FailOpen treats scanner errors and timeouts as an allow verdict
// instead of a block. Default is fail-closed, since the default
// posture for a security gate should block on infrastructure failure.
FailOpen bool `json:"fail_open" yaml:"fail_open"`
// Timeout bounds each scan call. Uses Go duration syntax (e.g. "30s").
// Default: "30s".
Timeout string `json:"timeout" yaml:"timeout"`
// SigningKey authenticates pull requests to the internal scan-fetch
// route used by every storage backend. Required whenever Enabled is
// true. Supports ${VAR_NAME} expansion like AuthConfig fields.
SigningKey string `json:"signing_key" yaml:"signing_key"`
// FetchBaseURL is the address scanners use to reach this proxy to pull
// staged artifacts. Defaults to BaseURL. Set this separately when
// scanners reach the proxy over an internal address different from the
// public-facing BaseURL (mirrors DirectServeBaseURL/UIBaseURL).
FetchBaseURL string `json:"fetch_base_url" yaml:"fetch_base_url"`
// Scanners is the list of external scanning services to call.
Scanners []ScannerConfig `json:"scanners" yaml:"scanners"`
}
// ScannerConfig configures a single external scanning service.
type ScannerConfig struct {
// Name identifies this scanner in logs and metrics.
Name string `json:"name" yaml:"name"`
// URL is the endpoint the proxy POSTs scan notifications to.
URL string `json:"url" yaml:"url"`
// Mode is "block" (default) or "monitor". A "block" scanner's verdict
// can prevent caching; a "monitor" scanner's findings are logged but
// never gate caching.
Mode string `json:"mode" yaml:"mode"`
// Ecosystems restricts this scanner to specific ecosystems (e.g.
// "npm", "pypi"). Empty means all ecosystems.
Ecosystems []string `json:"ecosystems" yaml:"ecosystems"`
// Headers are additional HTTP headers sent with every scan request
// (e.g. for authenticating to the scanner service). Values support
// ${VAR_NAME} expansion like AuthConfig fields.
Headers map[string]string `json:"headers" yaml:"headers"`
}
// SigningKeyExpanded returns SigningKey with ${VAR_NAME} references expanded.
func (s *ScanningConfig) SigningKeyExpanded() string {
return expandEnv(s.SigningKey)
}
// HeadersExpanded returns Headers with ${VAR_NAME} references expanded in
// each value.
func (s *ScannerConfig) HeadersExpanded() map[string]string {
if len(s.Headers) == 0 {
return nil
}
expanded := make(map[string]string, len(s.Headers))
for k, v := range s.Headers {
expanded[k] = expandEnv(v)
}
return expanded
}
// Validate checks the scanning configuration for errors, applying the
// default timeout if unset.
func (s *ScanningConfig) Validate() error {
if !s.Enabled {
return nil
}
if s.SigningKeyExpanded() == "" {
return fmt.Errorf("scanning.signing_key is required when scanning.enabled is true")
}
if len(s.Scanners) == 0 {
return fmt.Errorf("scanning.scanners must not be empty when scanning.enabled is true")
}
if s.FetchBaseURL != "" {
if err := validateAbsoluteURL("scanning.fetch_base_url", s.FetchBaseURL); err != nil {
return err
}
}
if s.Timeout == "" {
s.Timeout = defaultScanningTimeoutStr
}
if d, err := time.ParseDuration(s.Timeout); err != nil {
return fmt.Errorf("invalid scanning.timeout %q: %w", s.Timeout, err)
} else if d <= 0 {
return fmt.Errorf("invalid scanning.timeout %q: must be > 0", s.Timeout)
}
for i := range s.Scanners {
if err := s.Scanners[i].Validate(); err != nil {
return fmt.Errorf("scanning.scanners[%d]: %w", i, err)
}
}
return nil
}
// Validate checks a single scanner's configuration, applying the default
// mode ("block") if unset.
func (s *ScannerConfig) Validate() error {
if s.Name == "" {
return fmt.Errorf("name is required")
}
if err := validateAbsoluteURL("url", s.URL); err != nil {
return err
}
if s.Mode == "" {
s.Mode = "block"
}
switch s.Mode {
case "block", "monitor":
default:
return fmt.Errorf("invalid mode %q (must be block or monitor)", s.Mode)
}
return nil
}
2026-01-20 21:52:44 +00:00
// StorageConfig configures artifact storage.
type StorageConfig struct {
// URL is the storage backend URL.
// Supported schemes:
// - file:///path/to/dir - Local filesystem (default)
// - s3://bucket-name - Amazon S3
// - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
// - gs://bucket-name - Google Cloud Storage (Workload Identity supported)
// - azblob://container-name - Azure Blob Storage
// If empty, defaults to file:// with the Path value.
URL string `json:"url" yaml:"url"`
2026-01-20 21:52:44 +00:00
// Path is the directory where cached artifacts are stored.
// If URL is empty, this is used as file://{Path}.
//
// Deprecated: Use URL with file:// scheme instead.
2026-01-20 21:52:44 +00:00
Path string `json:"path" yaml:"path"`
// MaxSize is the maximum cache size (e.g., "10GB", "500MB").
// When exceeded, least recently used artifacts are evicted.
// Empty or "0" means unlimited.
MaxSize string `json:"max_size" yaml:"max_size"`
// DirectServe enables redirecting cached artifact downloads to presigned
// storage URLs (HTTP 302) instead of streaming bytes through the proxy.
// Only effective for backends that support URL signing (S3, GCS, Azure).
DirectServe bool `json:"direct_serve" yaml:"direct_serve"`
// DirectServeTTL is how long presigned URLs remain valid.
// Uses Go duration syntax (e.g. "5m", "1h"). Default: "15m".
DirectServeTTL string `json:"direct_serve_ttl" yaml:"direct_serve_ttl"`
// DirectServeBaseURL overrides the scheme and host of presigned URLs
// before returning them to clients. Useful when the proxy reaches
// storage at an internal address (e.g. 127.0.0.1 or a Docker hostname)
// but clients must use a public one.
DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"`
Add storage.cache_artifacts to serve artifacts without storing them (#387) * Add storage.passthrough to serve artifacts without storing them With storage.passthrough enabled the proxy streams every artifact from upstream to the client and never writes it to storage or the cache database. Metadata filtering, cooldown and the denylist work as before, so the proxy can sit behind another cache (e.g. an Artifactory remote) purely as a policy layer without holding a second copy of every package. Artifacts whose digest is known up front (OCI, Swift, Helm) are verified while streaming. Their responses are sent chunked and the connection is aborted on a mismatch, so a client never receives a tampered artifact as a complete response. Passthrough is rejected together with scanning, direct_serve and mirror_api, and the mirror command refuses to run with it, since all of them depend on stored artifacts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Address review: abort truncated streams, guard every cache read - serveArtifact aborts the response on any body read error, and when fewer bytes than the declared size were written, not only on a digest mismatch. A failed or short upstream body in passthrough mode no longer reaches the client as a complete 200. - Streamed upstream read failures are logged and counted as stream_failed upstream errors before the response is aborted. - checkCache reports a miss when artifact caching is off, so every cache read path, including Swift archive HEAD requests, ignores entries stored before the mode was enabled. Denylisted versions are still rejected. - Rename storage.passthrough to storage.cache_artifacts (default true, PROXY_STORAGE_CACHE_ARTIFACTS), which names what actually changes. Tests cover an OCI blob shorter than its Content-Length and an npm tarball whose chunked response ends without the final chunk, both served through the HTTP handlers, plus a Swift HEAD request against a cached archive. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Rename Proxy.Passthrough to StreamArtifacts Match the storage.cache_artifacts option: the field says what changes (artifacts are streamed instead of stored) and its zero value keeps the usual caching behaviour for every Proxy built without the server config, such as the mirror command and tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 15:28:20 +07:00
// CacheArtifacts stores fetched artifacts so later downloads are served
// from storage. When false, every download streams from upstream and
// nothing is stored; metadata is still cached, and cooldown and the
// denylist still apply. Useful when another caching layer sits in front
// of the proxy. False is incompatible with scanning, direct_serve and
// mirror_api, which all depend on stored artifacts. Default: true.
CacheArtifacts bool `json:"cache_artifacts" yaml:"cache_artifacts"`
Add age-based retention for cached artifacts (#428) * Add age-based retention for cached artifacts Cached artifacts stayed forever unless storage.max_size pushed them out. storage.retention adds a default, per-ecosystem and per-package duration after which an artifact nobody downloaded gets evicted. Ecosystems opt in one at a time: a handler registers with retention.Default, and until it does, configuring that ecosystem (or one of its packages) fails validation, and the default doesn't touch it. This commit registers none, so nothing changes yet. Startup logs which ecosystems the default covers and which it doesn't. An artifact counts as expired when both its fetch time and its last access are older than the cutoff. Checking only the last access would evict a refetched artifact again right away, since clearing a record keeps its old access time and a cache miss doesn't record a hit. The sweep scans artifacts in id windows, so no single query holds the SQLite connection for long, and checks each row against its own rule. Expired records are cleared and their files go through pending_deletes instead of being deleted directly, because a buffered hit can make an artifact that's being served right now look expired. Buffered hits are flushed before each sweep, and one sweep clears at most 10,000 artifacts. Since a big first sweep can queue a lot, reclaim now keeps working through due entries for up to 30 seconds per tick instead of stopping after 100, and pending_deletes gets an index on queued_at. Also: - proxy_artifacts_evicted_total{reason, ecosystem} counts LRU and retention evictions. - gradle.build_cache.max_age accepts "7d", as its comment always said. Refs #306 * Document storage retention Describes the new storage.retention block, the order rules apply in, that the default only covers ecosystems that support retention yet, and what to expect from the first sweep over an old cache. Also lists the new eviction counter and notes that gradle max_age takes days. Refs #306 * Fix retention timing on Postgres and tighten the sweep On Postgres, fetched_at and last_accessed_at are TIMESTAMP columns without a zone. They hold the local time the proxy wrote, but lib/pq reads them back labelled UTC. East of UTC that makes every artifact look hours younger than it is, so the sweep evicted late: two hours in Berlin, nine in Tokyo. The candidates' times are now read back as local time, and the database tests run on both SQLite and Postgres, with one that sets the local zone to Asia/Tokyo. A few smaller things: - When a delete fails, reclaim stops for the rest of the minute instead of trying every queued path. Otherwise a storage outage turns into thirty seconds of failed deletes and warnings every minute. - If the buffered hits can't be written, the sweep skips that round. Running on stale access times could clear something that is being downloaded right now. - DefaultCanonical only accepts a package key when rebuilding it gives the same PURL. For alpine and deb it would otherwise produce keys that never match. - sweep_interval has to be at least a minute. - New tests cover the eviction counter and an artifact that gets a hit between the scan and the clear. - The docs no longer say nothing changes without retention, and they point out that the example's ecosystem and package rules only pass validation once those ecosystems support retention. Refs #306 * Reject qualifiers and subpaths in retention package keys A rule names a whole package. Qualifiers like ?type=pom were dropped on the way to the stored PURL and the rule quietly covered every file of the package. Now such a key fails validation. Refs #306 * Say when a package key doesn't match how packages are stored A package rule for an ecosystem that supports retention, written in a form its packages aren't stored under (pkg:npm/babel/core without the @), failed with "retention is not supported for npm packages". Once the ecosystem is enabled that is no longer true. Such a key now fails with "does not match how npm packages are stored" and points to the package rule forms in the docs. Keys for ecosystems without retention support keep the old message. Refs #306 * Note that re-running the mirror keeps packages cached A mirror run over a package that is still cached goes through the normal cache lookup and records a download, so the package's age starts over. The docs only said that mirrored packages age from the time they were mirrored. Refs #306 * Say when an ecosystem takes no package rules Some ecosystems will support retention without package rules, because nothing in their stored records names a package the way a PURL would (conan, helm). A package key for one of them used to fail with "does not match how conan packages are stored; see the package rule forms", which sends people looking for a form that doesn't exist. It now says package rules aren't supported there and points to the ecosystem rule. Refs #306 * Use helm as the example of an ecosystem without package rules Conan gets package rules once its handler stores archives under the recipe name, so the config test that shows the refusal now uses helm, whose cache records name the repository, not the chart. Refs #306 * Say that each cached file ages on its own The retention docs said each artifact, each version of a package, ages on its own. A version can have several cached files (a jar and its pom, several wheels, Conan sources and binaries), and each of them ages separately. Refs #306 * Clear an expired artifact and queue its file in one transaction The sweep cleared the record and queued the file for deletion as two separate writes. If queueing failed, or the process stopped in between, the file stayed in storage with no record pointing at it and no queue entry, and no later sweep would find it again. Both writes now share a transaction, so a failed queue rolls the clear back and the next sweep retries. Refs #306 * Show artifact evictions on the analytics page proxy_artifacts_evicted_total had no place on /ui/analytics. The Runtime card now lists evicted artifacts by reason and ecosystem, without the failure tint, and the coverage test records an eviction so it doesn't depend on test order. Refs #306 * Store artifact times in UTC fetched_at and last_accessed_at are now written in UTC, and the retention cutoffs are compared in UTC, the same convention #434 uses for the metadata cache. lib/pq sends a time with its offset and a Postgres TIMESTAMP keeps only the wall clock, so UTC on write is what reads back correctly. That replaces reinterpreting the times as local on read. Rows written before this still hold local time. East of UTC they look younger by the offset until they are rewritten, so retention evicts them a few hours late once; west of UTC that much early. Refs #306 --------- Co-authored-by: t <t@t>
2026-10-12 10:06:01 +02:00
// Retention evicts cached artifacts that have not been accessed for a
// configured time, per ecosystem and per package.
Retention RetentionConfig `json:"retention" yaml:"retention"`
}
// RetentionConfig configures age-based eviction of cached artifacts. An
// artifact's age is the time since it was last served, or since it was
// fetched when it has not been served since. Durations accept a "d" suffix
// for days; "0" or empty means never evict by age.
type RetentionConfig struct {
// Default applies to every ecosystem that supports retention and has no
// rule of its own.
Default string `json:"default" yaml:"default"`
// Ecosystems overrides the default per ecosystem key (e.g. "npm", "oci").
// Configuring an ecosystem whose retention is not supported yet is an
// error.
Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"`
// Packages overrides the ecosystem rule for single packages, keyed by
// package PURL.
Packages map[string]string `json:"packages" yaml:"packages"`
// SweepInterval is how often expired artifacts are looked for, at least
// one minute. Default: "10m".
SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"`
2026-01-20 21:52:44 +00:00
}
// GradleConfig configures Gradle-specific features.
type GradleConfig struct {
// BuildCache configures the /gradle HttpBuildCache endpoint.
BuildCache GradleBuildCacheConfig `json:"build_cache" yaml:"build_cache"`
}
// GradleBuildCacheConfig configures Gradle HttpBuildCache safeguards.
type GradleBuildCacheConfig struct {
// ReadOnly disables PUT uploads and keeps cache reads (GET/HEAD) enabled.
ReadOnly bool `json:"read_only" yaml:"read_only"`
// MaxUploadSize caps a single PUT body size (e.g., "100MB"). Must be > 0.
// Default: "100MB".
MaxUploadSize string `json:"max_upload_size" yaml:"max_upload_size"`
// MaxAge evicts entries older than this duration (e.g., "24h", "7d").
// Empty or "0" disables age-based eviction.
MaxAge string `json:"max_age" yaml:"max_age"`
// MaxSize evicts oldest entries until total Gradle cache size is <= MaxSize.
// Empty or "0" disables size-based eviction.
MaxSize string `json:"max_size" yaml:"max_size"`
// SweepInterval controls periodic eviction frequency.
// Default: "10m".
SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"`
}
Add storage backend probe to /health (closes #73) (#119) * config: add Health.StorageProbeInterval * metrics: add proxy_health_probe_failures_total counter * server: add storageProbe with happy-path test * server: add storageProbe failure-mode tests * server: add healthCache with TTL, single-flight, transition logging * server: wire storage probe into /health * server: update TestHealthEndpoint for JSON; wire healthCache into newTestServer Also fix Windows file-locking issue in storageProbe: close the reader explicitly before Delete so the file handle is released prior to os.Remove. * server: clean up stale comment in storageProbe * docs: document storage health probe and new metric * docs: regenerate Swagger for /health JSON response * server: simplify rc.Close error handling in storageProbe * server: defer probe cleanup so size/open/read/verify failures don't leak objects Previously, storageProbe only called Delete on the success path. Any failure between Store and the final Delete (size mismatch, Open error, mid-stream read failure, content mismatch) left the probe object orphaned in the storage backend. With caching disabled and Kubernetes-rate probing, the leak could accumulate noticeably on backends like S3. Use a named return + defer to attempt Delete after every successful Store. The earlier-step failure remains the primary error; Delete failure only surfaces as step="delete" when nothing else went wrong. Add a table-driven test that asserts cleanup runs for each non-delete failure path. Reported by Copilot on #119. * config: validate health.storage_probe_interval in Config.Validate The new duration field was only validated at use time in newHealthCache. The existing codebase already validates other duration fields (MetadataTTL, DirectServeTTL, Gradle.MaxAge, Gradle.SweepInterval) in Config.Validate() so misconfiguration fails fast at startup with a config-key-specific error. Match that pattern. The parse-at-use code in newHealthCache stays as a safety net, mirroring the MetadataTTL precedent. Reported by Copilot on #119. * docs: lowercase "counter" in metrics table for consistency Other rows in the table use lowercase type names (counter/gauge/histogram). Match that style. Reported by Copilot on #119. * docs: include size-check step in /health probe description The probe is write → size-check → read → verify → delete; the architecture note was missing the size-check step. Reported by Copilot on #119. * server: address andrew's review on #119 - Drop unused callerCtx parameter from healthCache.Check (Check is now parameter-less; the comment-only "accepted for symmetry" justification wasn't carrying its weight). - Emit "storage": {"status": "skipped"} on DB short-circuit instead of omitting the key, so monitors expecting a fixed key set keep working. - Reject negative storage_probe_interval at config validation time (previously parsed and silently behaved like "0"). - Extract HealthConfig.Validate to keep Config.Validate under the gocognit threshold and match the existing GradleBuildCacheConfig pattern. - README Health Check section: note that /health is intended as a readiness probe rather than a liveness probe (Check holds a mutex for up to the 10s probe timeout). - cmd/proxy/main.go godoc: column-align the new env var with the surrounding Gradle entries. Reported by andrew on #119.
2026-05-22 14:14:01 +03:00
// HealthConfig configures the /health endpoint.
type HealthConfig struct {
// StorageProbeInterval is the minimum time between storage backend probes.
// Uses Go duration syntax (e.g. "30s", "1m"). Default: "30s".
// Set to "0" to probe on every /health request (useful for low-traffic deployments).
StorageProbeInterval string `json:"storage_probe_interval" yaml:"storage_probe_interval"`
}
2026-01-20 21:52:44 +00:00
// DatabaseConfig configures the cache database.
type DatabaseConfig struct {
// Driver is the database driver: "sqlite" or "postgres".
Driver string `json:"driver" yaml:"driver"`
2026-01-20 21:52:44 +00:00
// Path is the path to the SQLite database file.
Path string `json:"path" yaml:"path"`
// URL is the PostgreSQL connection string.
URL string `json:"url" yaml:"url"`
// HitFlushInterval is how often cache hit counts and last-access times
// are written, batched in one transaction. Uses Go duration syntax
// (e.g. "1s", "10s"). Default: "1s". Set to "0" to write each hit as it
// happens.
HitFlushInterval string `json:"hit_flush_interval" yaml:"hit_flush_interval"`
2026-01-20 21:52:44 +00:00
}
// String returns a human-readable description of the configured database
// suitable for logging. For postgres the password in the connection URL is
// redacted; if the URL cannot be parsed only the driver name is returned to
// avoid leaking credentials.
func (d DatabaseConfig) String() string {
if d.Driver == "postgres" {
u, err := url.Parse(d.URL)
if err != nil || u.Host == "" {
return "postgres"
}
return u.Redacted()
}
return d.Path
}
2026-01-20 21:52:44 +00:00
// LogConfig configures logging.
type LogConfig struct {
// Level is the minimum log level: "debug", "info", "warn", "error".
Level string `json:"level" yaml:"level"`
// Format is the log format: "text" or "json".
Format string `json:"format" yaml:"format"`
}
// AccessLogConfig configures the JSONL activity log.
type AccessLogConfig struct {
// Path is the file to append activity records to. Empty disables the access log.
Path string `json:"path" yaml:"path"`
}
// UpstreamConfig configures upstream URLs for built-in routes and authentication.
2026-01-20 21:52:44 +00:00
// Leave empty to use defaults.
type UpstreamConfig struct {
// AllowPrivateHosts permits listed upstream hosts to resolve to private addresses.
AllowPrivateHosts []string `json:"allow_private_hosts" yaml:"allow_private_hosts"`
// AllowLoopback permits upstream requests and redirects to loopback addresses.
AllowLoopback bool `json:"allow_loopback" yaml:"allow_loopback"`
2026-01-20 21:52:44 +00:00
// NPM is the upstream npm registry URL.
// Default: https://registry.npmjs.org
NPM string `json:"npm" yaml:"npm"`
// NPMFullMetadata always requests the full packument (application/json)
// from the npm upstream, so served metadata carries the "time" map even
// when cooldown is disabled. Clients that gate on publish age (for
// example Yarn's npmMinimalAgeGate) need this.
// Default: false (the abbreviated format is preferred).
NPMFullMetadata bool `json:"npm_full_metadata" yaml:"npm_full_metadata"`
Add package routes for private Composer and npm registries (#426) * Add Composer package routes for private repositories - Add upstream.composer_routes mapping package name patterns to repositories - Serve matching packages only from their route, with no fallback to the default repository - Qualify metadata and artifact cache keys with the route's upstream - Give routed versions an empty notification-url so installs are not reported to notify-batch - Document routes in configuration docs, example config and README * Add npm package routes for private registries - Add upstream.npm_routes mapping package name patterns to registries - Serve matching packages only from their route, with no fallback to upstream.npm - Resolve and validate tarball URLs against the route's registry - Qualify metadata and artifact cache keys with the route's upstream - Document npm routes and the audit endpoint limitation * Share routed artifact filename qualification between handlers * Match Composer routes without the ~dev suffix Packagist serves a package's dev versions from {name}~dev.json, and the handler matched routes against the name with ~dev still attached. An exact route for example/library therefore missed /p2/example/library~dev.json, and the private package's dev metadata came from the default repository. - Match routes against the package name without ~dev, keeping it in the upstream URL and the cache key so the two files stay apart - Add a regression test with an exact route * Keep routed npm publish times out of the shared versions row The download cooldown check stores publish times under the plain version PURL, which does not say which registry a version came from. A time read from the default registry was trusted for a private version of the same name, so a private release published an hour ago passed a seven-day cooldown when the public one was a month old. - Skip the stored row for routed packages, both reading and writing, and take the publish time from the route's metadata, which is cached under the route - This also stops a private time from holding back the public version, and a route's previous upstream from lending its times to the new one - Cover adding a route after a public download, the reverse order, and moving a route to another upstream
2026-10-11 19:06:18 +02:00
// NPMRoutes maps package name patterns to npm registry URLs. A package
// whose name matches a pattern is fetched only from that registry, never
// from upstream.npm, so a public package cannot stand in for it. Patterns
// use path.Match syntax, e.g. "@example/*"; the longest matching pattern
// wins.
NPMRoutes map[string]string `json:"npm_routes" yaml:"npm_routes"`
// Cargo is the upstream cargo index URL.
// Default: https://index.crates.io
Cargo string `json:"cargo" yaml:"cargo"`
// CargoDownload is the upstream cargo download URL.
// Default: https://static.crates.io/crates
CargoDownload string `json:"cargo_download" yaml:"cargo_download"`
// Gem is the upstream RubyGems registry URL.
// Default: https://rubygems.org
Gem string `json:"gem" yaml:"gem"`
// Go is the upstream Go module proxy URL.
// Default: https://proxy.golang.org
Go string `json:"go" yaml:"go"`
// GoSumDBs maps Go checksum database names to their base URLs, served at
// /go/sumdb/{name}/ so the go command reaches the database through
// GOPROXY. Unknown names get 404, which makes the go command contact the
// database directly.
// Default when empty: {"sum.golang.org": "https://sum.golang.org"}.
GoSumDBs map[string]string `json:"go_sumdbs" yaml:"go_sumdbs"`
// GoSumDBsOffline keeps answering for a checksum database from its
// cached signed tree, however old, when the database cannot be reached.
// Set this when clients have no route to the database of their own;
// otherwise the proxy stops answering once the cached tree is older than
// metadata_ttl, and the go command contacts the database directly.
// Requires cache_metadata. Default: false.
GoSumDBsOffline bool `json:"go_sumdbs_offline" yaml:"go_sumdbs_offline"`
// Hex is the upstream Hex repository URL.
// Default: https://repo.hex.pm
Hex string `json:"hex" yaml:"hex"`
// HexAPI is the upstream Hex API URL used for package timestamps.
// Default: https://hex.pm
HexAPI string `json:"hex_api" yaml:"hex_api"`
// Pub is the upstream pub registry URL.
// Default: https://pub.dev
Pub string `json:"pub" yaml:"pub"`
// PyPI is the upstream PyPI index and API URL.
// Default: https://pypi.org
PyPI string `json:"pypi" yaml:"pypi"`
// PyPIDownload is the upstream PyPI package download URL.
// Default: https://files.pythonhosted.org
PyPIDownload string `json:"pypi_download" yaml:"pypi_download"`
// Maven is the upstream Maven repository URL.
// Default: https://repo1.maven.org/maven2
Maven string `json:"maven" yaml:"maven"`
// GradlePluginPortal is the upstream Gradle Plugin Portal Maven URL.
// Used to resolve Gradle plugin marker artifacts.
// Default: https://plugins.gradle.org/m2
GradlePluginPortal string `json:"gradle_plugin_portal" yaml:"gradle_plugin_portal"`
// NuGet is the upstream NuGet API URL.
// Default: https://api.nuget.org
NuGet string `json:"nuget" yaml:"nuget"`
2026-01-20 21:52:44 +00:00
// NuGetSearch is the upstream NuGet search API URL.
// Default: https://azuresearch-usnc.nuget.org
NuGetSearch string `json:"nuget_search" yaml:"nuget_search"`
// Composer is the upstream Packagist API URL.
// Default: https://packagist.org
Composer string `json:"composer" yaml:"composer"`
// ComposerRepository is the upstream Packagist repository URL.
// Default: https://repo.packagist.org
ComposerRepository string `json:"composer_repository" yaml:"composer_repository"`
Add package routes for private Composer and npm registries (#426) * Add Composer package routes for private repositories - Add upstream.composer_routes mapping package name patterns to repositories - Serve matching packages only from their route, with no fallback to the default repository - Qualify metadata and artifact cache keys with the route's upstream - Give routed versions an empty notification-url so installs are not reported to notify-batch - Document routes in configuration docs, example config and README * Add npm package routes for private registries - Add upstream.npm_routes mapping package name patterns to registries - Serve matching packages only from their route, with no fallback to upstream.npm - Resolve and validate tarball URLs against the route's registry - Qualify metadata and artifact cache keys with the route's upstream - Document npm routes and the audit endpoint limitation * Share routed artifact filename qualification between handlers * Match Composer routes without the ~dev suffix Packagist serves a package's dev versions from {name}~dev.json, and the handler matched routes against the name with ~dev still attached. An exact route for example/library therefore missed /p2/example/library~dev.json, and the private package's dev metadata came from the default repository. - Match routes against the package name without ~dev, keeping it in the upstream URL and the cache key so the two files stay apart - Add a regression test with an exact route * Keep routed npm publish times out of the shared versions row The download cooldown check stores publish times under the plain version PURL, which does not say which registry a version came from. A time read from the default registry was trusted for a private version of the same name, so a private release published an hour ago passed a seven-day cooldown when the public one was a month old. - Skip the stored row for routed packages, both reading and writing, and take the publish time from the route's metadata, which is cached under the route - This also stops a private time from holding back the public version, and a route's previous upstream from lending its times to the new one - Cover adding a route after a public download, the reverse order, and moving a route to another upstream
2026-10-11 19:06:18 +02:00
// ComposerRoutes maps package name patterns to Composer repository URLs
// (the base that serves /p2/{vendor}/{package}.json). A package whose
// name matches a pattern is fetched only from that repository, never
// from upstream.composer_repository, so a public package cannot stand in
// for it. Patterns use path.Match syntax, e.g. "example/*"; the longest
// matching pattern wins.
ComposerRoutes map[string]string `json:"composer_routes" yaml:"composer_routes"`
// Conan is the upstream Conan registry URL.
// Default: https://center.conan.io
Conan string `json:"conan" yaml:"conan"`
// Conda is the upstream Conda channel base URL.
// Default: https://conda.anaconda.org
Conda string `json:"conda" yaml:"conda"`
// CRAN is the upstream CRAN mirror URL.
// Default: https://cloud.r-project.org
CRAN string `json:"cran" yaml:"cran"`
// Julia is the upstream Julia package server URL.
// Default: https://pkg.julialang.org
Julia string `json:"julia" yaml:"julia"`
// OCIDefault is the default upstream OCI registry URL.
// Default: https://registry-1.docker.io
OCIDefault string `json:"oci_default" yaml:"oci_default"`
// Swift is the upstream Swift Package Registry URL.
// Default: https://tuist.dev/api/registry/swift
Swift string `json:"swift" yaml:"swift"`
// Debian is the upstream APT repository base URL, served at /debian/.
// Example: http://archive.ubuntu.com/ubuntu would get Ubuntu.
// Default: http://deb.debian.org/debian
Debian string `json:"debian" yaml:"debian"`
// DebianRepositories maps repository names to additional APT repository
// base URLs, served at /debian/{name}/.
// Example: {"security": "https://security.debian.org/debian-security"}.
DebianRepositories map[string]string `json:"debian_repositories" yaml:"debian_repositories"`
// RPM is the upstream RPM repository base URL.
// Default: https://dl.fedoraproject.org/pub/fedora/linux
RPM string `json:"rpm" yaml:"rpm"`
2026-09-03 16:59:12 +01:00
// HomebrewAPI is the upstream Homebrew JSON API URL.
// Default: https://formulae.brew.sh/api
HomebrewAPI string `json:"homebrew_api" yaml:"homebrew_api"`
// HomebrewArtifact is the upstream registry URL for Homebrew artifacts.
// Default: https://ghcr.io
HomebrewArtifact string `json:"homebrew_artifact" yaml:"homebrew_artifact"`
// Helm maps repository names to HTTP Helm chart repository URLs.
// Requests use /helm/{name}/index.yaml and chart URLs in the index are
// rewritten to the same named proxy endpoint.
Helm map[string]string `json:"helm" yaml:"helm"`
// APK maps repository names to Alpine APK repository base URLs, served
// at /apk/{name}/. The remaining request path mirrors the upstream
// layout, e.g. /apk/alpine/v3.22/main/x86_64/APKINDEX.tar.gz.
// Default when empty: {"alpine": "https://dl-cdn.alpinelinux.org/alpine"}.
APK map[string]string `json:"apk" yaml:"apk"`
// Arch maps repository names to Arch Linux pacman mirror base URLs,
// served at /arch/{name}/. The remaining request path mirrors the
// upstream layout, e.g. /arch/archlinux/core/os/x86_64/core.db.
// Default when empty: {"archlinux": "https://geo.mirror.pkgbuild.com"}.
Arch map[string]string `json:"arch" yaml:"arch"`
// OCI maps names to OCI registry URLs. Requests to a named registry use
// the repository prefix upstream/{name}/, for example
// oci://proxy.example.com/upstream/ghcr/owner/chart.
OCI map[string]string `json:"oci" yaml:"oci"`
Route containerd ns mirror requests to configured registries (#405) * Route containerd ns mirror requests to configured registries containerd's hosts.toml mirrors append ?ns=<registry-host> to every request. The container handler ignored it, so a single _default mirror entry could not serve more than one registry. ns is a closed-world lookup key and is never dialed. Docker Hub aliases and the host of upstream.oci_default select the default route, hosts of upstream.oci entries select their named upstream. Unknown hosts return NAME_UNKNOWN so containerd falls back to its next host. Registry URLs with a path are not indexed, because ns names the registry at the root of a host. Host collisions log a warning and resolve deterministically. With ns the path is the verbatim upstream repository, the reserved upstream/ prefix is rejected, and repository prefix routes such as Homebrew's do not apply. Cache names match the unprefixed and upstream/{name}/ routes, so all routes share cached blobs, manifests and tag lists. ns is no longer forwarded on tag-list requests, and the pagination Link is rewritten per request instead of at store time so clients on different routes get links for their own route. Refs #303 * Document containerd ns mirroring Refs #303 * Version the tag-list cache identity for the raw-link format Tag-list rows now store the upstream Link verbatim and rewrite it per request. Rows written by earlier versions hold a Link already rewritten for one route. Under the unchanged key an older binary running next to this one (rolling update on shared Postgres, or a rollback) would serve the raw Link unrewritten, sending clients to the wrong route for the next page. A format marker in the key keeps the two apart at the cost of one cache miss per tag list after the upgrade. Adversarial review finding F1: tag-list cache rows hold raw upstream Links under unchanged keys, so an older binary serves them unrewritten. * Qualify the README claim about ns mirror coverage Registries whose URL has a path and hosts shared by two entries are not reachable through ns; point to the configuration guide for both rules. Adversarial review finding F3: README says ns covers every upstream.oci registry. * Redact registry URLs in ns index warnings Configured registry URLs may carry userinfo credentials, which the new startup warnings wrote to the log verbatim. Log them with the password masked, and say precisely what a path-prefixed default registry means: its host is not indexed, while the Docker Hub aliases still select it. Adversarial review finding F2: startup warnings print registry URLs unredacted and the default-registry message is misleading. * Keep non-default ports apart in the ns index Dropping 80 and 443 regardless of scheme turned https://host:80 and https://host into the same key, so a request for one of them could land on the other registry. Only the default port of the URL's scheme is optional now. A configured host is indexed with and without that port, because image references spell it either way; any other port has to match exactly, and the ns value is used as containerd sends it, apart from case and IPv6 brackets. Since a URL now yields two keys that can belong to different routes, the collision warning lists the owner per key. The handler test runs two registries on one host that differ only in the port. Tests can't bind 80 or 443, so a dialer maps those addresses to the fake servers. * Allow upstream/{name}/ together with ns when they agree containerd sends ns on every request to a mirror host, override_path or not. Per-registry mirrors pointing at /v2/upstream/{name} therefore arrive as upstream/{name}/...?ns=<registry>, and refusing every prefixed name under ns broke them, including the only way to mirror an upstream whose URL has a path. Such requests are now handled like the prefix route without ns, as long as ns names that upstream's own host under the same port rules as the index. Any other ns on a prefixed name is still NAME_UNKNOWN, so ns can't be used to create cache entries under another registry's name. * README: containerd config_path and how to check it A _default/hosts.toml does nothing while CRI has no hosts directory configured, so add the config.toml snippet for containerd 2.x and 1.x. To check the setup, look at containerd config dump and pull with crictl, then find the request in the proxy log; ctr --hosts-dir reads the directory on its own and would succeed even without config_path. Also mention that existing per-registry override_path entries keep working and that k3s generates the directory itself. * Let the prefix route take ns for aliases and mirrored registries With upstream.oci.hub pointing at registry-1.docker.io and a docker.io hosts.toml mirror on /v2/upstream/hub, containerd sends ns=docker.io, and the prefix check only accepted the upstream's own host. The same happens whenever the upstream is a mirror of the registry the nodes pull from, say an Artifactory remote for ghcr.io: ns carries ghcr.io, the upstream host is something else, and the pull got a 404. The prefix already decides where the content comes from and which cache entries are used, so ns can't redirect anything there. The check now only refuses an ns that contradicts the prefix, meaning a host the proxy knows that belongs to another route. The Docker Hub aliases count as one host, and a host the proxy doesn't know at all is accepted, since a client can only arrive at the prefix through its own mirror entry. * Take docker.io on every prefix route A Docker Hub mirror configured as a named upstream (mirror.gcr.io, an Artifactory remote) with a docker.io hosts.toml pointing at /v2/upstream/hub got a 404 again: docker.io is always in the index for the default route, so the check saw a known host of another route. That setup worked before this branch. Docker Hub repository names have exactly two path components, so docker.io/upstream/... can never be a real image, and accepting the aliases on every prefix route shadows nothing. While here: the collision warning claimed ns requests go to the other route, which is only true without the prefix, and the README said pulls for unknown registries always get a 404, which isn't the case for paths under upstream/{name}/. * Only take a foreign ns on a prefix route when it is configured A containerd _default mirror sends the same request for a pull of unconfigured.example/upstream/mirror/owner/app as a ghcr.io hosts.toml with override_path does for a mirror upstream. Accepting any host the proxy didn't know therefore let such a pull be answered with owner/app from the mirror upstream, a different image, instead of a 404 that lets containerd go to the registry the image actually names. The prefix route now accepts ns only for the upstream's own host, for Docker Hub, and for hosts listed in the new upstream.oci_mirrors setting. That setting is how an Artifactory remote of ghcr.io keeps working: upstream: oci_mirrors: ghcr: ["ghcr.io"] The listed hosts also select the upstream for unprefixed ns requests, which makes a mirror with a path in its URL reachable through the _default entry as well. Hosts from configured URLs still win over mirror entries; the proxy warns about such overlaps at startup. * Fix the docs sentence on which ns a prefix route takes configuration.md still said a prefix request with ns is only refused when ns names another route's host. Since the last commit it is the other way round: only the upstream's own host, its oci_mirrors entries and Docker Hub get through. * Note what a host accepted on a prefix route gives up Listing ghcr.io for an upstream (or the upstream's own host) still has the ambiguity from the wildcard case: ghcr.io/upstream/ghcr/owner/app lands on that upstream. That is the price of an explicit entry and fine in practice, but it should be written down. * Test that oci_mirrors entries stay with their own upstream Two gaps in the coverage of the last change: nothing failed if a host listed for one upstream was also accepted on another upstream's prefix, and nothing checked that the server actually hands upstream.oci_mirrors to the container handler. The handler test now has a second upstream that must refuse ghcr.io, and a small server test pulls through a mirror entry on both the prefix and the unprefixed route. * Log when a prefix route refuses an ns Setups with a ghcr.io hosts.toml pointing at a mirror upstream worked before this branch and now need an oci_mirrors entry. Without one, containerd gets a 404, quietly pulls from ghcr.io directly, and nobody notices the proxy is being skipped. The refusal is now logged as a warning that names the upstream and the ns and points at upstream.oci_mirrors. * Drop a check in validateOCIMirrors that could never fire user@ghcr.io parses with ghcr.io as its host, so the parsed.Host != host comparison already rejects it and the extra parsed.User test was dead. A test case for a host with credentials keeps that path covered. * Reject an oci_mirrors host that is listed for two upstreams With ghcr.io under both art and nexus, both prefix routes took ns=ghcr.io while unprefixed requests went to whichever name sorted first, with nothing but a startup warning. Config validation now refuses the second entry, comparing hosts without case and without the default :443. * Log each refused ns once per upstream and refuse overlong values The refusal warning fired on every request, and a node with a missing oci_mirrors entry asks for the manifest and each layer, so the same line piled up. Each upstream/ns pair is now logged once, for up to 1024 pairs; after that every refusal is logged again. ns comes straight from the client and can be as long as the request line allows, so remembering it as is would let anyone pin a lot of memory with a handful of requests. No registry host is longer than 259 characters (a 253-character name plus a port), so anything longer is refused right away, before it is logged or remembered. * Compare oci_mirrors hosts the way the handler does The duplicate check lowercased the host and trimmed :443, while the handler splits host and port, brackets IPv6 literals and drops an empty port. So ghcr.io and ghcr.io:, or [::1] and [::1]:, passed as different hosts in two upstreams and then collided in the handler. The check now builds the same key the handler uses. * Rewrap the oci_mirrors paragraphs in the configuration docs Two sentences had been dropped into the middle of wrapped lines, and the prefix-route paragraph named Docker Hub twice in a row. Same content, shorter lines, one mention. * Stop logging ns refusals once the set is full Past 1024 different upstream/ns pairs every refusal was logged again, so anyone could fill the set with made-up hosts and then flood the log at warning level. When the set is full the proxy now says so once and keeps quiet after that. A real misconfiguration shows up long before that, with the first pull. * Test the limit on remembered ns refusals Nothing covered what happens once 1024 pairs are remembered. The new test fills the set, then sends a known pair and two new ones: the proxy must log the limit notice exactly once, log none of the refusals and keep the set at its size.
2026-10-10 08:35:50 +02:00
// OCIMirrors lists, per upstream.oci name, the registry hosts that
// upstream mirrors, for example {"ghcr": ["ghcr.io"]} for an Artifactory
// remote of ghcr.io. containerd requests whose ns query parameter names
// one of these hosts are served by that upstream.
OCIMirrors map[string][]string `json:"oci_mirrors" yaml:"oci_mirrors"`
Add generic HTTP download proxy for GitHub release assets (mise/aqua) (#302) * Add generic HTTP download proxy for GitHub release assets Adds a /generic/{name}/ route backed by a new upstream.generic named-upstream map, so tools that download from fixed URL shapes (mise's aqua backend fetching GitHub release assets, and its tag lookups on api.github.com) can be pointed at the proxy with client-side URL rewriting. Only configured upstreams are reachable, so this is not an open HTTP proxy. Paths shaped like {owner}/{repo}/releases/download/{tag}/{asset} are version-pinned and go through the artifact cache: fetched once, hashed, served without revalidation, and still served when the upstream is down. Every other path goes through the metadata cache with the client's Accept header and query string replayed, so API responses are fresh within metadata_ttl, revalidated after that, and served stale when the upstream fails or rate-limits the request. Tests cover path classification, unknown upstreams and traversal, cache hits with the upstream down, HEAD, 404 pass-through, Accept and query forwarding, stale-on-429, cache isolation between upstreams, and that an upstream token scoped to the release host is not sent to the object store it redirects to. Closes #183. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RDjDeq27CKzEP2o3GWBY7F * Use fixed Accept header for generic metadata --------- Co-authored-by: Giles Westwood <giles@gileswestwood.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-03 16:44:54 +01:00
// Generic maps names to plain HTTP upstream base URLs, served at
// /generic/{name}/. The remaining request path and query string are
// appended to the upstream URL. GitHub release asset paths
// ({owner}/{repo}/releases/download/{tag}/{asset}) and paths matching
// GenericArtifacts are cached in the artifact cache; everything else
// goes through the metadata cache.
Add generic HTTP download proxy for GitHub release assets (mise/aqua) (#302) * Add generic HTTP download proxy for GitHub release assets Adds a /generic/{name}/ route backed by a new upstream.generic named-upstream map, so tools that download from fixed URL shapes (mise's aqua backend fetching GitHub release assets, and its tag lookups on api.github.com) can be pointed at the proxy with client-side URL rewriting. Only configured upstreams are reachable, so this is not an open HTTP proxy. Paths shaped like {owner}/{repo}/releases/download/{tag}/{asset} are version-pinned and go through the artifact cache: fetched once, hashed, served without revalidation, and still served when the upstream is down. Every other path goes through the metadata cache with the client's Accept header and query string replayed, so API responses are fresh within metadata_ttl, revalidated after that, and served stale when the upstream fails or rate-limits the request. Tests cover path classification, unknown upstreams and traversal, cache hits with the upstream down, HEAD, 404 pass-through, Accept and query forwarding, stale-on-429, cache isolation between upstreams, and that an upstream token scoped to the release host is not sent to the object store it redirects to. Closes #183. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RDjDeq27CKzEP2o3GWBY7F * Use fixed Accept header for generic metadata --------- Co-authored-by: Giles Westwood <giles@gileswestwood.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-03 16:44:54 +01:00
// Example: {"github": "https://github.com", "github-api": "https://api.github.com"}.
Generic map[string]string `json:"generic" yaml:"generic"`
// GenericArtifacts maps a Generic upstream name to regular expressions for
// its version-pinned download paths. A request path after
// /generic/{name}/ that one of them matches in full is cached in the
// artifact cache like a GitHub release asset. Each pattern needs `version`
// and `file` named groups; an optional `name` group names the package,
// which otherwise is the upstream name.
// Example: {"apache": ["dist/(?P<name>maven)/maven-3/(?P<version>[^/]+)/binaries/(?P<file>[^/]+)"]}.
GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"`
// Auth configures authentication for upstream registries.
// Keys are absolute URL scopes matched by scheme, host, effective port,
// and path-segment prefix.
// Example: "https://npm.pkg.github.com" matches all requests to that host.
Auth map[string]AuthConfig `json:"auth" yaml:"auth"`
}
// AuthForURL returns the auth config that matches the given URL.
// The longest matching URL scope wins.
func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig {
if u.Auth == nil {
return nil
}
target, err := parseAuthURL(url)
if err != nil {
return nil
}
var bestMatch *AuthConfig
var bestLen int
for pattern, auth := range u.Auth {
configured, err := parseAuthURL(pattern)
if err == nil && authURLMatches(configured, target) && len(pattern) > bestLen {
a := auth // copy to avoid loop variable capture
bestMatch = &a
bestLen = len(pattern)
}
}
return bestMatch
}
// Validate checks upstream authentication URL scopes.
func (u *UpstreamConfig) Validate() error {
for pattern := range u.Auth {
if _, err := parseAuthURL(pattern); err != nil {
return fmt.Errorf("invalid upstream.auth URL %q: %w", pattern, err)
}
}
if err := validateNamedUpstreams("upstream.helm", u.Helm); err != nil {
return err
}
Add package routes for private Composer and npm registries (#426) * Add Composer package routes for private repositories - Add upstream.composer_routes mapping package name patterns to repositories - Serve matching packages only from their route, with no fallback to the default repository - Qualify metadata and artifact cache keys with the route's upstream - Give routed versions an empty notification-url so installs are not reported to notify-batch - Document routes in configuration docs, example config and README * Add npm package routes for private registries - Add upstream.npm_routes mapping package name patterns to registries - Serve matching packages only from their route, with no fallback to upstream.npm - Resolve and validate tarball URLs against the route's registry - Qualify metadata and artifact cache keys with the route's upstream - Document npm routes and the audit endpoint limitation * Share routed artifact filename qualification between handlers * Match Composer routes without the ~dev suffix Packagist serves a package's dev versions from {name}~dev.json, and the handler matched routes against the name with ~dev still attached. An exact route for example/library therefore missed /p2/example/library~dev.json, and the private package's dev metadata came from the default repository. - Match routes against the package name without ~dev, keeping it in the upstream URL and the cache key so the two files stay apart - Add a regression test with an exact route * Keep routed npm publish times out of the shared versions row The download cooldown check stores publish times under the plain version PURL, which does not say which registry a version came from. A time read from the default registry was trusted for a private version of the same name, so a private release published an hour ago passed a seven-day cooldown when the public one was a month old. - Skip the stored row for routed packages, both reading and writing, and take the publish time from the route's metadata, which is cached under the route - This also stops a private time from holding back the public version, and a route's previous upstream from lending its times to the new one - Cover adding a route after a public download, the reverse order, and moving a route to another upstream
2026-10-11 19:06:18 +02:00
if err := validatePackageRoutes("upstream.composer_routes", u.ComposerRoutes); err != nil {
return err
}
if err := validatePackageRoutes("upstream.npm_routes", u.NPMRoutes); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.apk", u.APK); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.go_sumdbs", u.GoSumDBs); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil {
return err
}
Route containerd ns mirror requests to configured registries (#405) * Route containerd ns mirror requests to configured registries containerd's hosts.toml mirrors append ?ns=<registry-host> to every request. The container handler ignored it, so a single _default mirror entry could not serve more than one registry. ns is a closed-world lookup key and is never dialed. Docker Hub aliases and the host of upstream.oci_default select the default route, hosts of upstream.oci entries select their named upstream. Unknown hosts return NAME_UNKNOWN so containerd falls back to its next host. Registry URLs with a path are not indexed, because ns names the registry at the root of a host. Host collisions log a warning and resolve deterministically. With ns the path is the verbatim upstream repository, the reserved upstream/ prefix is rejected, and repository prefix routes such as Homebrew's do not apply. Cache names match the unprefixed and upstream/{name}/ routes, so all routes share cached blobs, manifests and tag lists. ns is no longer forwarded on tag-list requests, and the pagination Link is rewritten per request instead of at store time so clients on different routes get links for their own route. Refs #303 * Document containerd ns mirroring Refs #303 * Version the tag-list cache identity for the raw-link format Tag-list rows now store the upstream Link verbatim and rewrite it per request. Rows written by earlier versions hold a Link already rewritten for one route. Under the unchanged key an older binary running next to this one (rolling update on shared Postgres, or a rollback) would serve the raw Link unrewritten, sending clients to the wrong route for the next page. A format marker in the key keeps the two apart at the cost of one cache miss per tag list after the upgrade. Adversarial review finding F1: tag-list cache rows hold raw upstream Links under unchanged keys, so an older binary serves them unrewritten. * Qualify the README claim about ns mirror coverage Registries whose URL has a path and hosts shared by two entries are not reachable through ns; point to the configuration guide for both rules. Adversarial review finding F3: README says ns covers every upstream.oci registry. * Redact registry URLs in ns index warnings Configured registry URLs may carry userinfo credentials, which the new startup warnings wrote to the log verbatim. Log them with the password masked, and say precisely what a path-prefixed default registry means: its host is not indexed, while the Docker Hub aliases still select it. Adversarial review finding F2: startup warnings print registry URLs unredacted and the default-registry message is misleading. * Keep non-default ports apart in the ns index Dropping 80 and 443 regardless of scheme turned https://host:80 and https://host into the same key, so a request for one of them could land on the other registry. Only the default port of the URL's scheme is optional now. A configured host is indexed with and without that port, because image references spell it either way; any other port has to match exactly, and the ns value is used as containerd sends it, apart from case and IPv6 brackets. Since a URL now yields two keys that can belong to different routes, the collision warning lists the owner per key. The handler test runs two registries on one host that differ only in the port. Tests can't bind 80 or 443, so a dialer maps those addresses to the fake servers. * Allow upstream/{name}/ together with ns when they agree containerd sends ns on every request to a mirror host, override_path or not. Per-registry mirrors pointing at /v2/upstream/{name} therefore arrive as upstream/{name}/...?ns=<registry>, and refusing every prefixed name under ns broke them, including the only way to mirror an upstream whose URL has a path. Such requests are now handled like the prefix route without ns, as long as ns names that upstream's own host under the same port rules as the index. Any other ns on a prefixed name is still NAME_UNKNOWN, so ns can't be used to create cache entries under another registry's name. * README: containerd config_path and how to check it A _default/hosts.toml does nothing while CRI has no hosts directory configured, so add the config.toml snippet for containerd 2.x and 1.x. To check the setup, look at containerd config dump and pull with crictl, then find the request in the proxy log; ctr --hosts-dir reads the directory on its own and would succeed even without config_path. Also mention that existing per-registry override_path entries keep working and that k3s generates the directory itself. * Let the prefix route take ns for aliases and mirrored registries With upstream.oci.hub pointing at registry-1.docker.io and a docker.io hosts.toml mirror on /v2/upstream/hub, containerd sends ns=docker.io, and the prefix check only accepted the upstream's own host. The same happens whenever the upstream is a mirror of the registry the nodes pull from, say an Artifactory remote for ghcr.io: ns carries ghcr.io, the upstream host is something else, and the pull got a 404. The prefix already decides where the content comes from and which cache entries are used, so ns can't redirect anything there. The check now only refuses an ns that contradicts the prefix, meaning a host the proxy knows that belongs to another route. The Docker Hub aliases count as one host, and a host the proxy doesn't know at all is accepted, since a client can only arrive at the prefix through its own mirror entry. * Take docker.io on every prefix route A Docker Hub mirror configured as a named upstream (mirror.gcr.io, an Artifactory remote) with a docker.io hosts.toml pointing at /v2/upstream/hub got a 404 again: docker.io is always in the index for the default route, so the check saw a known host of another route. That setup worked before this branch. Docker Hub repository names have exactly two path components, so docker.io/upstream/... can never be a real image, and accepting the aliases on every prefix route shadows nothing. While here: the collision warning claimed ns requests go to the other route, which is only true without the prefix, and the README said pulls for unknown registries always get a 404, which isn't the case for paths under upstream/{name}/. * Only take a foreign ns on a prefix route when it is configured A containerd _default mirror sends the same request for a pull of unconfigured.example/upstream/mirror/owner/app as a ghcr.io hosts.toml with override_path does for a mirror upstream. Accepting any host the proxy didn't know therefore let such a pull be answered with owner/app from the mirror upstream, a different image, instead of a 404 that lets containerd go to the registry the image actually names. The prefix route now accepts ns only for the upstream's own host, for Docker Hub, and for hosts listed in the new upstream.oci_mirrors setting. That setting is how an Artifactory remote of ghcr.io keeps working: upstream: oci_mirrors: ghcr: ["ghcr.io"] The listed hosts also select the upstream for unprefixed ns requests, which makes a mirror with a path in its URL reachable through the _default entry as well. Hosts from configured URLs still win over mirror entries; the proxy warns about such overlaps at startup. * Fix the docs sentence on which ns a prefix route takes configuration.md still said a prefix request with ns is only refused when ns names another route's host. Since the last commit it is the other way round: only the upstream's own host, its oci_mirrors entries and Docker Hub get through. * Note what a host accepted on a prefix route gives up Listing ghcr.io for an upstream (or the upstream's own host) still has the ambiguity from the wildcard case: ghcr.io/upstream/ghcr/owner/app lands on that upstream. That is the price of an explicit entry and fine in practice, but it should be written down. * Test that oci_mirrors entries stay with their own upstream Two gaps in the coverage of the last change: nothing failed if a host listed for one upstream was also accepted on another upstream's prefix, and nothing checked that the server actually hands upstream.oci_mirrors to the container handler. The handler test now has a second upstream that must refuse ghcr.io, and a small server test pulls through a mirror entry on both the prefix and the unprefixed route. * Log when a prefix route refuses an ns Setups with a ghcr.io hosts.toml pointing at a mirror upstream worked before this branch and now need an oci_mirrors entry. Without one, containerd gets a 404, quietly pulls from ghcr.io directly, and nobody notices the proxy is being skipped. The refusal is now logged as a warning that names the upstream and the ns and points at upstream.oci_mirrors. * Drop a check in validateOCIMirrors that could never fire user@ghcr.io parses with ghcr.io as its host, so the parsed.Host != host comparison already rejects it and the extra parsed.User test was dead. A test case for a host with credentials keeps that path covered. * Reject an oci_mirrors host that is listed for two upstreams With ghcr.io under both art and nexus, both prefix routes took ns=ghcr.io while unprefixed requests went to whichever name sorted first, with nothing but a startup warning. Config validation now refuses the second entry, comparing hosts without case and without the default :443. * Log each refused ns once per upstream and refuse overlong values The refusal warning fired on every request, and a node with a missing oci_mirrors entry asks for the manifest and each layer, so the same line piled up. Each upstream/ns pair is now logged once, for up to 1024 pairs; after that every refusal is logged again. ns comes straight from the client and can be as long as the request line allows, so remembering it as is would let anyone pin a lot of memory with a handful of requests. No registry host is longer than 259 characters (a 253-character name plus a port), so anything longer is refused right away, before it is logged or remembered. * Compare oci_mirrors hosts the way the handler does The duplicate check lowercased the host and trimmed :443, while the handler splits host and port, brackets IPv6 literals and drops an empty port. So ghcr.io and ghcr.io:, or [::1] and [::1]:, passed as different hosts in two upstreams and then collided in the handler. The check now builds the same key the handler uses. * Rewrap the oci_mirrors paragraphs in the configuration docs Two sentences had been dropped into the middle of wrapped lines, and the prefix-route paragraph named Docker Hub twice in a row. Same content, shorter lines, one mention. * Stop logging ns refusals once the set is full Past 1024 different upstream/ns pairs every refusal was logged again, so anyone could fill the set with made-up hosts and then flood the log at warning level. When the set is full the proxy now says so once and keeps quiet after that. A real misconfiguration shows up long before that, with the first pull. * Test the limit on remembered ns refusals Nothing covered what happens once 1024 pairs are remembered. The new test fills the set, then sends a known pair and two new ones: the proxy must log the limit notice exactly once, log none of the refusals and keep the set at its size.
2026-10-10 08:35:50 +02:00
if err := validateOCIMirrors(u.OCIMirrors, u.OCI); err != nil {
return err
}
Add generic HTTP download proxy for GitHub release assets (mise/aqua) (#302) * Add generic HTTP download proxy for GitHub release assets Adds a /generic/{name}/ route backed by a new upstream.generic named-upstream map, so tools that download from fixed URL shapes (mise's aqua backend fetching GitHub release assets, and its tag lookups on api.github.com) can be pointed at the proxy with client-side URL rewriting. Only configured upstreams are reachable, so this is not an open HTTP proxy. Paths shaped like {owner}/{repo}/releases/download/{tag}/{asset} are version-pinned and go through the artifact cache: fetched once, hashed, served without revalidation, and still served when the upstream is down. Every other path goes through the metadata cache with the client's Accept header and query string replayed, so API responses are fresh within metadata_ttl, revalidated after that, and served stale when the upstream fails or rate-limits the request. Tests cover path classification, unknown upstreams and traversal, cache hits with the upstream down, HEAD, 404 pass-through, Accept and query forwarding, stale-on-429, cache isolation between upstreams, and that an upstream token scoped to the release host is not sent to the object store it redirects to. Closes #183. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RDjDeq27CKzEP2o3GWBY7F * Use fixed Accept header for generic metadata --------- Co-authored-by: Giles Westwood <giles@gileswestwood.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-03 16:44:54 +01:00
if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil {
return err
}
if err := u.validateGenericArtifacts(); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.debian_repositories", u.DebianRepositories); err != nil {
return err
}
for _, name := range debianReservedRepositoryNames {
if _, found := u.DebianRepositories[name]; found {
return fmt.Errorf(
"invalid upstream.debian_repositories name %q: reserved for the upstream.debian archive",
name,
)
}
}
return nil
}
func (u *UpstreamConfig) validateGenericArtifacts() error {
for name, patterns := range u.GenericArtifacts {
if _, found := u.Generic[name]; !found {
return fmt.Errorf("invalid upstream.generic_artifacts name %q: no upstream.generic entry", name)
}
for _, pattern := range patterns {
if _, err := CompileGenericArtifactPattern(pattern); err != nil {
return fmt.Errorf("invalid upstream.generic_artifacts.%s pattern %q: %w", name, pattern, err)
}
}
}
return nil
}
// validateUpstream checks the upstream block, including the one setting in
// it that depends on a top-level option.
func (c *Config) validateUpstream() error {
if err := c.Upstream.Validate(); err != nil {
return err
}
if c.Upstream.GoSumDBsOffline && !c.CacheMetadata {
return fmt.Errorf("upstream.go_sumdbs_offline requires cache_metadata")
}
return nil
}
// CompileGenericArtifactPattern compiles an upstream.generic_artifacts pattern
// so that it only matches a whole path, and checks it has the `version` and
// `file` groups.
func CompileGenericArtifactPattern(pattern string) (*regexp.Regexp, error) {
re, err := regexp.Compile(`^(?:` + pattern + `)$`)
if err != nil {
return nil, err
}
for _, group := range []string{"version", "file"} {
if re.SubexpIndex(group) < 0 {
return nil, fmt.Errorf("missing (?P<%s>...) group", group)
}
}
return re, nil
}
// debianReservedRepositoryNames are the upstream.debian archive's own root
// paths, which a repository of the same name would shadow.
var debianReservedRepositoryNames = []string{"pool", "dists"}
Route containerd ns mirror requests to configured registries (#405) * Route containerd ns mirror requests to configured registries containerd's hosts.toml mirrors append ?ns=<registry-host> to every request. The container handler ignored it, so a single _default mirror entry could not serve more than one registry. ns is a closed-world lookup key and is never dialed. Docker Hub aliases and the host of upstream.oci_default select the default route, hosts of upstream.oci entries select their named upstream. Unknown hosts return NAME_UNKNOWN so containerd falls back to its next host. Registry URLs with a path are not indexed, because ns names the registry at the root of a host. Host collisions log a warning and resolve deterministically. With ns the path is the verbatim upstream repository, the reserved upstream/ prefix is rejected, and repository prefix routes such as Homebrew's do not apply. Cache names match the unprefixed and upstream/{name}/ routes, so all routes share cached blobs, manifests and tag lists. ns is no longer forwarded on tag-list requests, and the pagination Link is rewritten per request instead of at store time so clients on different routes get links for their own route. Refs #303 * Document containerd ns mirroring Refs #303 * Version the tag-list cache identity for the raw-link format Tag-list rows now store the upstream Link verbatim and rewrite it per request. Rows written by earlier versions hold a Link already rewritten for one route. Under the unchanged key an older binary running next to this one (rolling update on shared Postgres, or a rollback) would serve the raw Link unrewritten, sending clients to the wrong route for the next page. A format marker in the key keeps the two apart at the cost of one cache miss per tag list after the upgrade. Adversarial review finding F1: tag-list cache rows hold raw upstream Links under unchanged keys, so an older binary serves them unrewritten. * Qualify the README claim about ns mirror coverage Registries whose URL has a path and hosts shared by two entries are not reachable through ns; point to the configuration guide for both rules. Adversarial review finding F3: README says ns covers every upstream.oci registry. * Redact registry URLs in ns index warnings Configured registry URLs may carry userinfo credentials, which the new startup warnings wrote to the log verbatim. Log them with the password masked, and say precisely what a path-prefixed default registry means: its host is not indexed, while the Docker Hub aliases still select it. Adversarial review finding F2: startup warnings print registry URLs unredacted and the default-registry message is misleading. * Keep non-default ports apart in the ns index Dropping 80 and 443 regardless of scheme turned https://host:80 and https://host into the same key, so a request for one of them could land on the other registry. Only the default port of the URL's scheme is optional now. A configured host is indexed with and without that port, because image references spell it either way; any other port has to match exactly, and the ns value is used as containerd sends it, apart from case and IPv6 brackets. Since a URL now yields two keys that can belong to different routes, the collision warning lists the owner per key. The handler test runs two registries on one host that differ only in the port. Tests can't bind 80 or 443, so a dialer maps those addresses to the fake servers. * Allow upstream/{name}/ together with ns when they agree containerd sends ns on every request to a mirror host, override_path or not. Per-registry mirrors pointing at /v2/upstream/{name} therefore arrive as upstream/{name}/...?ns=<registry>, and refusing every prefixed name under ns broke them, including the only way to mirror an upstream whose URL has a path. Such requests are now handled like the prefix route without ns, as long as ns names that upstream's own host under the same port rules as the index. Any other ns on a prefixed name is still NAME_UNKNOWN, so ns can't be used to create cache entries under another registry's name. * README: containerd config_path and how to check it A _default/hosts.toml does nothing while CRI has no hosts directory configured, so add the config.toml snippet for containerd 2.x and 1.x. To check the setup, look at containerd config dump and pull with crictl, then find the request in the proxy log; ctr --hosts-dir reads the directory on its own and would succeed even without config_path. Also mention that existing per-registry override_path entries keep working and that k3s generates the directory itself. * Let the prefix route take ns for aliases and mirrored registries With upstream.oci.hub pointing at registry-1.docker.io and a docker.io hosts.toml mirror on /v2/upstream/hub, containerd sends ns=docker.io, and the prefix check only accepted the upstream's own host. The same happens whenever the upstream is a mirror of the registry the nodes pull from, say an Artifactory remote for ghcr.io: ns carries ghcr.io, the upstream host is something else, and the pull got a 404. The prefix already decides where the content comes from and which cache entries are used, so ns can't redirect anything there. The check now only refuses an ns that contradicts the prefix, meaning a host the proxy knows that belongs to another route. The Docker Hub aliases count as one host, and a host the proxy doesn't know at all is accepted, since a client can only arrive at the prefix through its own mirror entry. * Take docker.io on every prefix route A Docker Hub mirror configured as a named upstream (mirror.gcr.io, an Artifactory remote) with a docker.io hosts.toml pointing at /v2/upstream/hub got a 404 again: docker.io is always in the index for the default route, so the check saw a known host of another route. That setup worked before this branch. Docker Hub repository names have exactly two path components, so docker.io/upstream/... can never be a real image, and accepting the aliases on every prefix route shadows nothing. While here: the collision warning claimed ns requests go to the other route, which is only true without the prefix, and the README said pulls for unknown registries always get a 404, which isn't the case for paths under upstream/{name}/. * Only take a foreign ns on a prefix route when it is configured A containerd _default mirror sends the same request for a pull of unconfigured.example/upstream/mirror/owner/app as a ghcr.io hosts.toml with override_path does for a mirror upstream. Accepting any host the proxy didn't know therefore let such a pull be answered with owner/app from the mirror upstream, a different image, instead of a 404 that lets containerd go to the registry the image actually names. The prefix route now accepts ns only for the upstream's own host, for Docker Hub, and for hosts listed in the new upstream.oci_mirrors setting. That setting is how an Artifactory remote of ghcr.io keeps working: upstream: oci_mirrors: ghcr: ["ghcr.io"] The listed hosts also select the upstream for unprefixed ns requests, which makes a mirror with a path in its URL reachable through the _default entry as well. Hosts from configured URLs still win over mirror entries; the proxy warns about such overlaps at startup. * Fix the docs sentence on which ns a prefix route takes configuration.md still said a prefix request with ns is only refused when ns names another route's host. Since the last commit it is the other way round: only the upstream's own host, its oci_mirrors entries and Docker Hub get through. * Note what a host accepted on a prefix route gives up Listing ghcr.io for an upstream (or the upstream's own host) still has the ambiguity from the wildcard case: ghcr.io/upstream/ghcr/owner/app lands on that upstream. That is the price of an explicit entry and fine in practice, but it should be written down. * Test that oci_mirrors entries stay with their own upstream Two gaps in the coverage of the last change: nothing failed if a host listed for one upstream was also accepted on another upstream's prefix, and nothing checked that the server actually hands upstream.oci_mirrors to the container handler. The handler test now has a second upstream that must refuse ghcr.io, and a small server test pulls through a mirror entry on both the prefix and the unprefixed route. * Log when a prefix route refuses an ns Setups with a ghcr.io hosts.toml pointing at a mirror upstream worked before this branch and now need an oci_mirrors entry. Without one, containerd gets a 404, quietly pulls from ghcr.io directly, and nobody notices the proxy is being skipped. The refusal is now logged as a warning that names the upstream and the ns and points at upstream.oci_mirrors. * Drop a check in validateOCIMirrors that could never fire user@ghcr.io parses with ghcr.io as its host, so the parsed.Host != host comparison already rejects it and the extra parsed.User test was dead. A test case for a host with credentials keeps that path covered. * Reject an oci_mirrors host that is listed for two upstreams With ghcr.io under both art and nexus, both prefix routes took ns=ghcr.io while unprefixed requests went to whichever name sorted first, with nothing but a startup warning. Config validation now refuses the second entry, comparing hosts without case and without the default :443. * Log each refused ns once per upstream and refuse overlong values The refusal warning fired on every request, and a node with a missing oci_mirrors entry asks for the manifest and each layer, so the same line piled up. Each upstream/ns pair is now logged once, for up to 1024 pairs; after that every refusal is logged again. ns comes straight from the client and can be as long as the request line allows, so remembering it as is would let anyone pin a lot of memory with a handful of requests. No registry host is longer than 259 characters (a 253-character name plus a port), so anything longer is refused right away, before it is logged or remembered. * Compare oci_mirrors hosts the way the handler does The duplicate check lowercased the host and trimmed :443, while the handler splits host and port, brackets IPv6 literals and drops an empty port. So ghcr.io and ghcr.io:, or [::1] and [::1]:, passed as different hosts in two upstreams and then collided in the handler. The check now builds the same key the handler uses. * Rewrap the oci_mirrors paragraphs in the configuration docs Two sentences had been dropped into the middle of wrapped lines, and the prefix-route paragraph named Docker Hub twice in a row. Same content, shorter lines, one mention. * Stop logging ns refusals once the set is full Past 1024 different upstream/ns pairs every refusal was logged again, so anyone could fill the set with made-up hosts and then flood the log at warning level. When the set is full the proxy now says so once and keeps quiet after that. A real misconfiguration shows up long before that, with the first pull. * Test the limit on remembered ns refusals Nothing covered what happens once 1024 pairs are remembered. The new test fills the set, then sends a known pair and two new ones: the proxy must log the limit notice exactly once, log none of the refusals and keep the set at its size.
2026-10-10 08:35:50 +02:00
// validateOCIMirrors checks that every upstream.oci_mirrors entry belongs to
// an upstream.oci name and lists bare registry hosts, the form containerd
// sends in the ns query parameter. A host may belong to one upstream only;
// otherwise both prefixes would take it while unprefixed requests silently
// went to one of them.
func validateOCIMirrors(mirrors map[string][]string, upstreams map[string]string) error {
names := make([]string, 0, len(mirrors))
for name := range mirrors {
names = append(names, name)
}
sort.Strings(names)
owners := make(map[string]string)
for _, name := range names {
if _, ok := upstreams[name]; !ok {
return fmt.Errorf("invalid upstream.oci_mirrors name %q: no upstream.oci entry of that name", name)
}
for _, host := range mirrors[name] {
parsed, err := url.Parse("https://" + host)
if host == "" || err != nil || parsed.Host != host {
return fmt.Errorf("invalid upstream.oci_mirrors.%s host %q: must be a registry host such as ghcr.io or registry.example:5000", name, host)
}
key := ociMirrorHostKey(host)
if owner, taken := owners[key]; taken && owner != name {
return fmt.Errorf("invalid upstream.oci_mirrors.%s host %q: already listed for %s", name, host, owner)
}
owners[key] = name
}
}
return nil
}
// ociMirrorHostKey normalizes an upstream.oci_mirrors host the way the
// container handler builds its ns lookup keys: lowercase, IPv6 literals in
// brackets, an empty or default https port dropped. Two entries with the same
// key would select the same route.
func ociMirrorHostKey(host string) string {
name, port, err := net.SplitHostPort(host)
if err != nil {
name, port = strings.TrimSuffix(strings.TrimPrefix(host, "["), "]"), ""
}
name = strings.ToLower(name)
if strings.Contains(name, ":") {
name = "[" + name + "]"
}
if port == "" || port == "443" {
return name
}
return name + ":" + port
}
func validateNamedUpstreams(field string, upstreams map[string]string) error {
for name, upstreamURL := range upstreams {
if name == "" || name == "." || name == ".." || strings.ContainsAny(name, `/\\`) {
return fmt.Errorf("invalid %s name %q", field, name)
}
if err := validateAbsoluteURL(field+"."+name, upstreamURL); err != nil {
Add package routes for private Composer and npm registries (#426) * Add Composer package routes for private repositories - Add upstream.composer_routes mapping package name patterns to repositories - Serve matching packages only from their route, with no fallback to the default repository - Qualify metadata and artifact cache keys with the route's upstream - Give routed versions an empty notification-url so installs are not reported to notify-batch - Document routes in configuration docs, example config and README * Add npm package routes for private registries - Add upstream.npm_routes mapping package name patterns to registries - Serve matching packages only from their route, with no fallback to upstream.npm - Resolve and validate tarball URLs against the route's registry - Qualify metadata and artifact cache keys with the route's upstream - Document npm routes and the audit endpoint limitation * Share routed artifact filename qualification between handlers * Match Composer routes without the ~dev suffix Packagist serves a package's dev versions from {name}~dev.json, and the handler matched routes against the name with ~dev still attached. An exact route for example/library therefore missed /p2/example/library~dev.json, and the private package's dev metadata came from the default repository. - Match routes against the package name without ~dev, keeping it in the upstream URL and the cache key so the two files stay apart - Add a regression test with an exact route * Keep routed npm publish times out of the shared versions row The download cooldown check stores publish times under the plain version PURL, which does not say which registry a version came from. A time read from the default registry was trusted for a private version of the same name, so a private release published an hour ago passed a seven-day cooldown when the public one was a month old. - Skip the stored row for routed packages, both reading and writing, and take the publish time from the route's metadata, which is cached under the route - This also stops a private time from holding back the public version, and a route's previous upstream from lending its times to the new one - Cover adding a route after a public download, the reverse order, and moving a route to another upstream
2026-10-11 19:06:18 +02:00
return err
}
}
return nil
}
func validatePackageRoutes(field string, routes map[string]string) error {
for pattern, upstreamURL := range routes {
if _, err := path.Match(pattern, ""); pattern == "" || err != nil {
return fmt.Errorf("invalid %s pattern %q", field, pattern)
}
if err := validateAbsoluteURL(field+"."+pattern, upstreamURL); err != nil {
return err
}
}
return nil
}
func parseAuthURL(value string) (*url.URL, error) {
parsed, err := url.Parse(value)
if err != nil || !parsed.IsAbs() || parsed.Hostname() == "" || parsed.Opaque != "" {
return nil, fmt.Errorf("invalid authentication URL")
}
return parsed, nil
}
func authURLMatches(configured, target *url.URL) bool {
if !strings.EqualFold(configured.Scheme, target.Scheme) ||
!strings.EqualFold(configured.Hostname(), target.Hostname()) ||
authURLPort(configured) != authURLPort(target) {
return false
}
if configured.RawQuery != "" && configured.RawQuery != target.RawQuery {
return false
}
configuredPath := strings.TrimSuffix(configured.EscapedPath(), "/")
if configuredPath == "" {
return true
}
targetPath := strings.TrimSuffix(target.EscapedPath(), "/")
return targetPath == configuredPath || strings.HasPrefix(targetPath, configuredPath+"/")
}
func authURLPort(value *url.URL) string {
if port := value.Port(); port != "" {
return port
}
if strings.EqualFold(value.Scheme, "https") {
return "443"
}
if strings.EqualFold(value.Scheme, "http") {
return "80"
}
return ""
}
// AuthConfig configures authentication for an upstream registry.
type AuthConfig struct {
// Type is the authentication type: "bearer", "basic", "header", or "ecr".
Type string `json:"type" yaml:"type"`
// Token is used for bearer authentication.
// Can reference environment variables with ${VAR_NAME} syntax.
Token string `json:"token" yaml:"token"`
// Username is used for basic authentication.
Username string `json:"username" yaml:"username"`
// Password is used for basic authentication.
// Can reference environment variables with ${VAR_NAME} syntax.
Password string `json:"password" yaml:"password"`
// HeaderName is the custom header name (for type "header").
HeaderName string `json:"header_name" yaml:"header_name"`
// HeaderValue is the custom header value (for type "header").
// Can reference environment variables with ${VAR_NAME} syntax.
HeaderValue string `json:"header_value" yaml:"header_value"`
// Region is the AWS region for ECR authentication (for type "ecr").
// If empty, the region is inferred from private ECR registry URLs before
// falling back to the AWS SDK default region chain.
Region string `json:"region" yaml:"region"`
2026-01-20 21:52:44 +00:00
}
// Default returns a Config with sensible defaults.
func Default() *Config {
return &Config{
Listen: ":8080",
BaseURL: "http://localhost:8080",
Storage: StorageConfig{
Add storage.cache_artifacts to serve artifacts without storing them (#387) * Add storage.passthrough to serve artifacts without storing them With storage.passthrough enabled the proxy streams every artifact from upstream to the client and never writes it to storage or the cache database. Metadata filtering, cooldown and the denylist work as before, so the proxy can sit behind another cache (e.g. an Artifactory remote) purely as a policy layer without holding a second copy of every package. Artifacts whose digest is known up front (OCI, Swift, Helm) are verified while streaming. Their responses are sent chunked and the connection is aborted on a mismatch, so a client never receives a tampered artifact as a complete response. Passthrough is rejected together with scanning, direct_serve and mirror_api, and the mirror command refuses to run with it, since all of them depend on stored artifacts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Address review: abort truncated streams, guard every cache read - serveArtifact aborts the response on any body read error, and when fewer bytes than the declared size were written, not only on a digest mismatch. A failed or short upstream body in passthrough mode no longer reaches the client as a complete 200. - Streamed upstream read failures are logged and counted as stream_failed upstream errors before the response is aborted. - checkCache reports a miss when artifact caching is off, so every cache read path, including Swift archive HEAD requests, ignores entries stored before the mode was enabled. Denylisted versions are still rejected. - Rename storage.passthrough to storage.cache_artifacts (default true, PROXY_STORAGE_CACHE_ARTIFACTS), which names what actually changes. Tests cover an OCI blob shorter than its Content-Length and an npm tarball whose chunked response ends without the final chunk, both served through the HTTP handlers, plus a Swift HEAD request against a cached archive. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Rename Proxy.Passthrough to StreamArtifacts Match the storage.cache_artifacts option: the field says what changes (artifacts are streamed instead of stored) and its zero value keeps the usual caching behaviour for every Proxy built without the server config, such as the mirror command and tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 15:28:20 +07:00
Path: "./cache/artifacts",
MaxSize: "",
CacheArtifacts: true,
Add age-based retention for cached artifacts (#428) * Add age-based retention for cached artifacts Cached artifacts stayed forever unless storage.max_size pushed them out. storage.retention adds a default, per-ecosystem and per-package duration after which an artifact nobody downloaded gets evicted. Ecosystems opt in one at a time: a handler registers with retention.Default, and until it does, configuring that ecosystem (or one of its packages) fails validation, and the default doesn't touch it. This commit registers none, so nothing changes yet. Startup logs which ecosystems the default covers and which it doesn't. An artifact counts as expired when both its fetch time and its last access are older than the cutoff. Checking only the last access would evict a refetched artifact again right away, since clearing a record keeps its old access time and a cache miss doesn't record a hit. The sweep scans artifacts in id windows, so no single query holds the SQLite connection for long, and checks each row against its own rule. Expired records are cleared and their files go through pending_deletes instead of being deleted directly, because a buffered hit can make an artifact that's being served right now look expired. Buffered hits are flushed before each sweep, and one sweep clears at most 10,000 artifacts. Since a big first sweep can queue a lot, reclaim now keeps working through due entries for up to 30 seconds per tick instead of stopping after 100, and pending_deletes gets an index on queued_at. Also: - proxy_artifacts_evicted_total{reason, ecosystem} counts LRU and retention evictions. - gradle.build_cache.max_age accepts "7d", as its comment always said. Refs #306 * Document storage retention Describes the new storage.retention block, the order rules apply in, that the default only covers ecosystems that support retention yet, and what to expect from the first sweep over an old cache. Also lists the new eviction counter and notes that gradle max_age takes days. Refs #306 * Fix retention timing on Postgres and tighten the sweep On Postgres, fetched_at and last_accessed_at are TIMESTAMP columns without a zone. They hold the local time the proxy wrote, but lib/pq reads them back labelled UTC. East of UTC that makes every artifact look hours younger than it is, so the sweep evicted late: two hours in Berlin, nine in Tokyo. The candidates' times are now read back as local time, and the database tests run on both SQLite and Postgres, with one that sets the local zone to Asia/Tokyo. A few smaller things: - When a delete fails, reclaim stops for the rest of the minute instead of trying every queued path. Otherwise a storage outage turns into thirty seconds of failed deletes and warnings every minute. - If the buffered hits can't be written, the sweep skips that round. Running on stale access times could clear something that is being downloaded right now. - DefaultCanonical only accepts a package key when rebuilding it gives the same PURL. For alpine and deb it would otherwise produce keys that never match. - sweep_interval has to be at least a minute. - New tests cover the eviction counter and an artifact that gets a hit between the scan and the clear. - The docs no longer say nothing changes without retention, and they point out that the example's ecosystem and package rules only pass validation once those ecosystems support retention. Refs #306 * Reject qualifiers and subpaths in retention package keys A rule names a whole package. Qualifiers like ?type=pom were dropped on the way to the stored PURL and the rule quietly covered every file of the package. Now such a key fails validation. Refs #306 * Say when a package key doesn't match how packages are stored A package rule for an ecosystem that supports retention, written in a form its packages aren't stored under (pkg:npm/babel/core without the @), failed with "retention is not supported for npm packages". Once the ecosystem is enabled that is no longer true. Such a key now fails with "does not match how npm packages are stored" and points to the package rule forms in the docs. Keys for ecosystems without retention support keep the old message. Refs #306 * Note that re-running the mirror keeps packages cached A mirror run over a package that is still cached goes through the normal cache lookup and records a download, so the package's age starts over. The docs only said that mirrored packages age from the time they were mirrored. Refs #306 * Say when an ecosystem takes no package rules Some ecosystems will support retention without package rules, because nothing in their stored records names a package the way a PURL would (conan, helm). A package key for one of them used to fail with "does not match how conan packages are stored; see the package rule forms", which sends people looking for a form that doesn't exist. It now says package rules aren't supported there and points to the ecosystem rule. Refs #306 * Use helm as the example of an ecosystem without package rules Conan gets package rules once its handler stores archives under the recipe name, so the config test that shows the refusal now uses helm, whose cache records name the repository, not the chart. Refs #306 * Say that each cached file ages on its own The retention docs said each artifact, each version of a package, ages on its own. A version can have several cached files (a jar and its pom, several wheels, Conan sources and binaries), and each of them ages separately. Refs #306 * Clear an expired artifact and queue its file in one transaction The sweep cleared the record and queued the file for deletion as two separate writes. If queueing failed, or the process stopped in between, the file stayed in storage with no record pointing at it and no queue entry, and no later sweep would find it again. Both writes now share a transaction, so a failed queue rolls the clear back and the next sweep retries. Refs #306 * Show artifact evictions on the analytics page proxy_artifacts_evicted_total had no place on /ui/analytics. The Runtime card now lists evicted artifacts by reason and ecosystem, without the failure tint, and the coverage test records an eviction so it doesn't depend on test order. Refs #306 * Store artifact times in UTC fetched_at and last_accessed_at are now written in UTC, and the retention cutoffs are compared in UTC, the same convention #434 uses for the metadata cache. lib/pq sends a time with its offset and a Postgres TIMESTAMP keeps only the wall clock, so UTC on write is what reads back correctly. That replaces reinterpreting the times as local on read. Rows written before this still hold local time. East of UTC they look younger by the offset until they are rewritten, so retention evicts them a few hours late once; west of UTC that much early. Refs #306 --------- Co-authored-by: t <t@t>
2026-10-12 10:06:01 +02:00
Retention: RetentionConfig{
SweepInterval: defaultRetentionSweepIntervalStr,
},
2026-01-20 21:52:44 +00:00
},
Database: DatabaseConfig{
Driver: "sqlite",
Path: "./cache/proxy.db",
2026-01-20 21:52:44 +00:00
},
Log: LogConfig{
Level: "info",
Format: "text",
},
Upstream: UpstreamConfig{
NPM: "https://registry.npmjs.org",
Cargo: "https://index.crates.io",
CargoDownload: "https://static.crates.io/crates",
Gem: "https://rubygems.org",
Go: "https://proxy.golang.org",
Hex: "https://repo.hex.pm",
HexAPI: "https://hex.pm",
Pub: "https://pub.dev",
PyPI: "https://pypi.org",
PyPIDownload: "https://files.pythonhosted.org",
Maven: "https://repo1.maven.org/maven2",
GradlePluginPortal: "https://plugins.gradle.org/m2",
NuGet: "https://api.nuget.org",
NuGetSearch: "https://azuresearch-usnc.nuget.org",
Composer: "https://packagist.org",
ComposerRepository: "https://repo.packagist.org",
Conan: "https://center.conan.io",
Conda: "https://conda.anaconda.org",
CRAN: "https://cloud.r-project.org",
Julia: "https://pkg.julialang.org",
Swift: DefaultSwiftUpstream,
OCIDefault: "https://registry-1.docker.io",
Debian: "http://deb.debian.org/debian",
RPM: "https://dl.fedoraproject.org/pub/fedora/linux",
2026-09-03 16:59:12 +01:00
HomebrewAPI: "https://formulae.brew.sh/api",
HomebrewArtifact: "https://ghcr.io",
2026-01-20 21:52:44 +00:00
},
Gradle: GradleConfig{
BuildCache: GradleBuildCacheConfig{
ReadOnly: false,
MaxUploadSize: defaultGradleMaxUploadSizeStr,
MaxAge: "168h",
MaxSize: "",
SweepInterval: defaultGradleSweepIntervalStr,
},
},
2026-01-20 21:52:44 +00:00
}
}
// Load reads configuration from a file (YAML or JSON).
func Load(path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("reading config file: %w", err)
}
cfg := Default()
ext := strings.ToLower(filepath.Ext(path))
switch ext {
case ".yaml", ".yml":
if err := yaml.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("parsing YAML config: %w", err)
}
case ".json":
if err := json.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("parsing JSON config: %w", err)
}
default:
// Try YAML first, then JSON
if err := yaml.Unmarshal(data, cfg); err != nil {
if err := json.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("parsing config (tried YAML and JSON): %w", err)
}
}
}
return cfg, nil
}
// setEnvString sets *dst from the named environment variable, leaving it
// untouched if the variable is unset or empty.
func setEnvString(dst *string, key string) {
if v := os.Getenv(key); v != "" {
*dst = v
}
}
// setEnvBool is setEnvString for boolean fields, parsed via envBool.
func setEnvBool(dst *bool, key string) {
if v := os.Getenv(key); v != "" {
*dst = envBool(v)
}
}
func setEnvStringSlice(dst *[]string, key string) {
value := os.Getenv(key)
if value == "" {
return
}
var items []string
for item := range strings.SplitSeq(value, ",") {
if item = strings.TrimSpace(item); item != "" {
items = append(items, item)
}
}
if len(items) > 0 {
*dst = items
}
}
2026-01-20 21:52:44 +00:00
// LoadFromEnv applies environment variable overrides to a Config.
// Environment variables use the PROXY_ prefix:
// - PROXY_LISTEN
// - PROXY_BASE_URL
// - PROXY_UI_URL
2026-01-20 21:52:44 +00:00
// - PROXY_STORAGE_PATH
// - PROXY_STORAGE_MAX_SIZE
// - PROXY_DATABASE_PATH
// - PROXY_LOG_LEVEL
// - PROXY_LOG_FORMAT
// - PROXY_ACCESS_LOG_PATH
// - PROXY_TRUST_FORWARDED_FOR
// - PROXY_UI_REQUEST_SOURCES
// - PROXY_UPSTREAM_SWIFT
Add storage backend probe to /health (closes #73) (#119) * config: add Health.StorageProbeInterval * metrics: add proxy_health_probe_failures_total counter * server: add storageProbe with happy-path test * server: add storageProbe failure-mode tests * server: add healthCache with TTL, single-flight, transition logging * server: wire storage probe into /health * server: update TestHealthEndpoint for JSON; wire healthCache into newTestServer Also fix Windows file-locking issue in storageProbe: close the reader explicitly before Delete so the file handle is released prior to os.Remove. * server: clean up stale comment in storageProbe * docs: document storage health probe and new metric * docs: regenerate Swagger for /health JSON response * server: simplify rc.Close error handling in storageProbe * server: defer probe cleanup so size/open/read/verify failures don't leak objects Previously, storageProbe only called Delete on the success path. Any failure between Store and the final Delete (size mismatch, Open error, mid-stream read failure, content mismatch) left the probe object orphaned in the storage backend. With caching disabled and Kubernetes-rate probing, the leak could accumulate noticeably on backends like S3. Use a named return + defer to attempt Delete after every successful Store. The earlier-step failure remains the primary error; Delete failure only surfaces as step="delete" when nothing else went wrong. Add a table-driven test that asserts cleanup runs for each non-delete failure path. Reported by Copilot on #119. * config: validate health.storage_probe_interval in Config.Validate The new duration field was only validated at use time in newHealthCache. The existing codebase already validates other duration fields (MetadataTTL, DirectServeTTL, Gradle.MaxAge, Gradle.SweepInterval) in Config.Validate() so misconfiguration fails fast at startup with a config-key-specific error. Match that pattern. The parse-at-use code in newHealthCache stays as a safety net, mirroring the MetadataTTL precedent. Reported by Copilot on #119. * docs: lowercase "counter" in metrics table for consistency Other rows in the table use lowercase type names (counter/gauge/histogram). Match that style. Reported by Copilot on #119. * docs: include size-check step in /health probe description The probe is write → size-check → read → verify → delete; the architecture note was missing the size-check step. Reported by Copilot on #119. * server: address andrew's review on #119 - Drop unused callerCtx parameter from healthCache.Check (Check is now parameter-less; the comment-only "accepted for symmetry" justification wasn't carrying its weight). - Emit "storage": {"status": "skipped"} on DB short-circuit instead of omitting the key, so monitors expecting a fixed key set keep working. - Reject negative storage_probe_interval at config validation time (previously parsed and silently behaved like "0"). - Extract HealthConfig.Validate to keep Config.Validate under the gocognit threshold and match the existing GradleBuildCacheConfig pattern. - README Health Check section: note that /health is intended as a readiness probe rather than a liveness probe (Check holds a mutex for up to the 10s probe timeout). - cmd/proxy/main.go godoc: column-align the new env var with the surrounding Gradle entries. Reported by andrew on #119.
2026-05-22 14:14:01 +03:00
// - PROXY_HEALTH_STORAGE_PROBE_INTERVAL
2026-01-20 21:52:44 +00:00
func (c *Config) LoadFromEnv() {
setEnvString(&c.Listen, "PROXY_LISTEN")
setEnvString(&c.BaseURL, "PROXY_BASE_URL")
setEnvString(&c.UIBaseURL, "PROXY_UI_URL")
setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR")
setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES")
setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS")
setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL")
setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH")
setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE")
setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE")
setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL")
setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL")
Add storage.cache_artifacts to serve artifacts without storing them (#387) * Add storage.passthrough to serve artifacts without storing them With storage.passthrough enabled the proxy streams every artifact from upstream to the client and never writes it to storage or the cache database. Metadata filtering, cooldown and the denylist work as before, so the proxy can sit behind another cache (e.g. an Artifactory remote) purely as a policy layer without holding a second copy of every package. Artifacts whose digest is known up front (OCI, Swift, Helm) are verified while streaming. Their responses are sent chunked and the connection is aborted on a mismatch, so a client never receives a tampered artifact as a complete response. Passthrough is rejected together with scanning, direct_serve and mirror_api, and the mirror command refuses to run with it, since all of them depend on stored artifacts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Address review: abort truncated streams, guard every cache read - serveArtifact aborts the response on any body read error, and when fewer bytes than the declared size were written, not only on a digest mismatch. A failed or short upstream body in passthrough mode no longer reaches the client as a complete 200. - Streamed upstream read failures are logged and counted as stream_failed upstream errors before the response is aborted. - checkCache reports a miss when artifact caching is off, so every cache read path, including Swift archive HEAD requests, ignores entries stored before the mode was enabled. Denylisted versions are still rejected. - Rename storage.passthrough to storage.cache_artifacts (default true, PROXY_STORAGE_CACHE_ARTIFACTS), which names what actually changes. Tests cover an OCI blob shorter than its Content-Length and an npm tarball whose chunked response ends without the final chunk, both served through the HTTP handlers, plus a Swift HEAD request against a cached archive. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Rename Proxy.Passthrough to StreamArtifacts Match the storage.cache_artifacts option: the field says what changes (artifacts are streamed instead of stored) and its zero value keeps the usual caching behaviour for every Proxy built without the server config, such as the mirror command and tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 15:28:20 +07:00
setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS")
Add age-based retention for cached artifacts (#428) * Add age-based retention for cached artifacts Cached artifacts stayed forever unless storage.max_size pushed them out. storage.retention adds a default, per-ecosystem and per-package duration after which an artifact nobody downloaded gets evicted. Ecosystems opt in one at a time: a handler registers with retention.Default, and until it does, configuring that ecosystem (or one of its packages) fails validation, and the default doesn't touch it. This commit registers none, so nothing changes yet. Startup logs which ecosystems the default covers and which it doesn't. An artifact counts as expired when both its fetch time and its last access are older than the cutoff. Checking only the last access would evict a refetched artifact again right away, since clearing a record keeps its old access time and a cache miss doesn't record a hit. The sweep scans artifacts in id windows, so no single query holds the SQLite connection for long, and checks each row against its own rule. Expired records are cleared and their files go through pending_deletes instead of being deleted directly, because a buffered hit can make an artifact that's being served right now look expired. Buffered hits are flushed before each sweep, and one sweep clears at most 10,000 artifacts. Since a big first sweep can queue a lot, reclaim now keeps working through due entries for up to 30 seconds per tick instead of stopping after 100, and pending_deletes gets an index on queued_at. Also: - proxy_artifacts_evicted_total{reason, ecosystem} counts LRU and retention evictions. - gradle.build_cache.max_age accepts "7d", as its comment always said. Refs #306 * Document storage retention Describes the new storage.retention block, the order rules apply in, that the default only covers ecosystems that support retention yet, and what to expect from the first sweep over an old cache. Also lists the new eviction counter and notes that gradle max_age takes days. Refs #306 * Fix retention timing on Postgres and tighten the sweep On Postgres, fetched_at and last_accessed_at are TIMESTAMP columns without a zone. They hold the local time the proxy wrote, but lib/pq reads them back labelled UTC. East of UTC that makes every artifact look hours younger than it is, so the sweep evicted late: two hours in Berlin, nine in Tokyo. The candidates' times are now read back as local time, and the database tests run on both SQLite and Postgres, with one that sets the local zone to Asia/Tokyo. A few smaller things: - When a delete fails, reclaim stops for the rest of the minute instead of trying every queued path. Otherwise a storage outage turns into thirty seconds of failed deletes and warnings every minute. - If the buffered hits can't be written, the sweep skips that round. Running on stale access times could clear something that is being downloaded right now. - DefaultCanonical only accepts a package key when rebuilding it gives the same PURL. For alpine and deb it would otherwise produce keys that never match. - sweep_interval has to be at least a minute. - New tests cover the eviction counter and an artifact that gets a hit between the scan and the clear. - The docs no longer say nothing changes without retention, and they point out that the example's ecosystem and package rules only pass validation once those ecosystems support retention. Refs #306 * Reject qualifiers and subpaths in retention package keys A rule names a whole package. Qualifiers like ?type=pom were dropped on the way to the stored PURL and the rule quietly covered every file of the package. Now such a key fails validation. Refs #306 * Say when a package key doesn't match how packages are stored A package rule for an ecosystem that supports retention, written in a form its packages aren't stored under (pkg:npm/babel/core without the @), failed with "retention is not supported for npm packages". Once the ecosystem is enabled that is no longer true. Such a key now fails with "does not match how npm packages are stored" and points to the package rule forms in the docs. Keys for ecosystems without retention support keep the old message. Refs #306 * Note that re-running the mirror keeps packages cached A mirror run over a package that is still cached goes through the normal cache lookup and records a download, so the package's age starts over. The docs only said that mirrored packages age from the time they were mirrored. Refs #306 * Say when an ecosystem takes no package rules Some ecosystems will support retention without package rules, because nothing in their stored records names a package the way a PURL would (conan, helm). A package key for one of them used to fail with "does not match how conan packages are stored; see the package rule forms", which sends people looking for a form that doesn't exist. It now says package rules aren't supported there and points to the ecosystem rule. Refs #306 * Use helm as the example of an ecosystem without package rules Conan gets package rules once its handler stores archives under the recipe name, so the config test that shows the refusal now uses helm, whose cache records name the repository, not the chart. Refs #306 * Say that each cached file ages on its own The retention docs said each artifact, each version of a package, ages on its own. A version can have several cached files (a jar and its pom, several wheels, Conan sources and binaries), and each of them ages separately. Refs #306 * Clear an expired artifact and queue its file in one transaction The sweep cleared the record and queued the file for deletion as two separate writes. If queueing failed, or the process stopped in between, the file stayed in storage with no record pointing at it and no queue entry, and no later sweep would find it again. Both writes now share a transaction, so a failed queue rolls the clear back and the next sweep retries. Refs #306 * Show artifact evictions on the analytics page proxy_artifacts_evicted_total had no place on /ui/analytics. The Runtime card now lists evicted artifacts by reason and ecosystem, without the failure tint, and the coverage test records an eviction so it doesn't depend on test order. Refs #306 * Store artifact times in UTC fetched_at and last_accessed_at are now written in UTC, and the retention cutoffs are compared in UTC, the same convention #434 uses for the metadata cache. lib/pq sends a time with its offset and a Postgres TIMESTAMP keeps only the wall clock, so UTC on write is what reads back correctly. That replaces reinterpreting the times as local on read. Rows written before this still hold local time. East of UTC they look younger by the offset until they are rewritten, so retention evicts them a few hours late once; west of UTC that much early. Refs #306 --------- Co-authored-by: t <t@t>
2026-10-12 10:06:01 +02:00
setEnvString(&c.Storage.Retention.Default, "PROXY_STORAGE_RETENTION_DEFAULT")
setEnvString(&c.Storage.Retention.SweepInterval, "PROXY_STORAGE_RETENTION_SWEEP_INTERVAL")
setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER")
setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH")
setEnvString(&c.Database.URL, "PROXY_DATABASE_URL")
setEnvString(&c.Database.HitFlushInterval, "PROXY_DATABASE_HIT_FLUSH_INTERVAL")
setEnvString(&c.Log.Level, "PROXY_LOG_LEVEL")
setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT")
setEnvString(&c.AccessLog.Path, "PROXY_ACCESS_LOG_PATH")
setEnvStringSlice(&c.Upstream.AllowPrivateHosts, "PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS")
setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK")
setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM")
setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA")
setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE")
setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO")
setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD")
setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM")
setEnvString(&c.Upstream.Go, "PROXY_UPSTREAM_GO")
setEnvString(&c.Upstream.Hex, "PROXY_UPSTREAM_HEX")
setEnvString(&c.Upstream.HexAPI, "PROXY_UPSTREAM_HEX_API")
setEnvString(&c.Upstream.Pub, "PROXY_UPSTREAM_PUB")
setEnvString(&c.Upstream.PyPI, "PROXY_UPSTREAM_PYPI")
setEnvString(&c.Upstream.PyPIDownload, "PROXY_UPSTREAM_PYPI_DOWNLOAD")
setEnvString(&c.Upstream.Maven, "PROXY_UPSTREAM_MAVEN")
setEnvString(&c.Upstream.GradlePluginPortal, "PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL")
setEnvString(&c.Upstream.NuGet, "PROXY_UPSTREAM_NUGET")
setEnvString(&c.Upstream.NuGetSearch, "PROXY_UPSTREAM_NUGET_SEARCH")
setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER")
setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY")
setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN")
setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA")
setEnvString(&c.Upstream.CRAN, "PROXY_UPSTREAM_CRAN")
setEnvString(&c.Upstream.Julia, "PROXY_UPSTREAM_JULIA")
setEnvString(&c.Upstream.Swift, "PROXY_UPSTREAM_SWIFT")
setEnvString(&c.Upstream.OCIDefault, "PROXY_UPSTREAM_OCI_DEFAULT")
setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN")
setEnvString(&c.Upstream.RPM, "PROXY_UPSTREAM_RPM")
2026-09-03 16:59:12 +01:00
setEnvString(&c.Upstream.HomebrewAPI, "PROXY_UPSTREAM_HOMEBREW_API")
setEnvString(&c.Upstream.HomebrewArtifact, "PROXY_UPSTREAM_HOMEBREW_ARTIFACT")
setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT")
Add pre-cache artifact scanning hook (trivy/ClamAV/Wiz/custom) (#298) Runs fetched artifacts through pluggable external scanners after they're staged in storage but before they're committed to the cache DB, so a block verdict deletes the object instead of ever exposing it to a client. Scanners pull the staged bytes themselves via a short-lived HMAC-signed internal route rather than the proxy pushing bytes to them, keeping the mechanism storage-backend-agnostic and avoiding uploading potentially huge artifacts through the proxy's own egress. Hardening baked in from the start: the internal scan-fetch route is gated both at router-mount time and in the handler so it's inert whenever scanning is disabled or unsigned; the signing key is mandatory whenever scanning is enabled, enforced directly in scanner.NewGroup rather than relying on callers to invoke config validation; the scan call and the delete-on-block cleanup both run on a context detached from the client's, so a client disconnecting mid-scan can't be mistaken for a scanner failure, doesn't cause a legitimate artifact to be deleted, and doesn't leave a genuinely blocked artifact's bytes orphaned in storage; and scanner infrastructure errors (connection failures, internal hostnames) are never forwarded verbatim to anonymous clients, only a generic message. The scan-error metric also correctly distinguishes a scanner's own timeout from being cancelled because a sibling scanner already decided the verdict. Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-03 05:06:44 -04:00
setEnvBool(&c.Scanning.Enabled, "PROXY_SCANNING_ENABLED")
setEnvBool(&c.Scanning.FailOpen, "PROXY_SCANNING_FAIL_OPEN")
setEnvString(&c.Scanning.Timeout, "PROXY_SCANNING_TIMEOUT")
setEnvString(&c.Scanning.SigningKey, "PROXY_SCANNING_SIGNING_KEY")
setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL")
setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA")
setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API")
setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED")
setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN")
setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE")
setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES")
setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS")
setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL")
setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE")
setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE")
Cache rewritten npm and Composer metadata (#402) The npm and Composer handlers rewrite every metadata document they serve so that download URLs point at the proxy: decode the whole document into generic maps, change the URLs, encode it again, and for Composer expand the minified format first. That ran on every request, cached metadata included. With metadata caching on and upstream out of the picture, a cached request still cost 7 ms for @babel/core, 17 ms for symfony/console and 155 ms and 134 MB of allocations for typescript, and an 8 vCPU VM serving cached npm packuments to 20 clients ran the proxy at 450% CPU for about 1,100 requests a second. Rewritten documents are now kept in memory, keyed by the ecosystem, proxy URL, package and a SHA-256 of the raw document, so new bytes from upstream are rewritten again and nothing is served stale. Requests that arrive while a document is being rewritten wait for that rewrite rather than running their own; a waiter leaves when its client does, and the rewrite still completes and is cached. The denylist is fixed at startup, so it needs no place in the key. Cooldown filtering depends on the current time, so with cooldown on the cache is bypassed. metadata_rewrite_cache_size bounds the cache (default "256MB", least recently used out first, "0" to rewrite on every request). NewProxy callers keep the old behaviour unless they set it. Whole cached requests, measured locally: @babel/core 7.1 ms -> 0.38 ms, 54,489 -> 119 allocations typescript 155 ms -> 10.5 ms, 1.37M -> 133 allocations symfony/console 17.4 ms -> 0.48 ms, 165,909 -> 123 allocations What remains is reading the raw document from storage and hashing it.
2026-10-03 04:09:15 -07:00
setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE")
setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT")
setEnvBool(&c.Gradle.BuildCache.ReadOnly, "PROXY_GRADLE_BUILD_CACHE_READ_ONLY")
setEnvString(&c.Gradle.BuildCache.MaxUploadSize, "PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE")
setEnvString(&c.Gradle.BuildCache.MaxAge, "PROXY_GRADLE_BUILD_CACHE_MAX_AGE")
setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE")
setEnvString(&c.Gradle.BuildCache.SweepInterval, "PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL")
setEnvString(&c.Health.StorageProbeInterval, "PROXY_HEALTH_STORAGE_PROBE_INTERVAL")
2026-01-20 21:52:44 +00:00
}
// validateAbsoluteURL returns an error if value is not a parseable URL with
// both a scheme and host. fieldName is used in the error message.
func validateAbsoluteURL(fieldName, value string) error {
u, err := url.Parse(value)
if err != nil || u.Scheme == "" || u.Host == "" {
return fmt.Errorf("invalid %s %q: must be an absolute URL", fieldName, value)
}
return nil
}
2026-01-20 21:52:44 +00:00
// Validate checks the configuration for errors.
func (c *Config) Validate() error {
if c.Listen == "" {
return fmt.Errorf("listen address is required")
}
if c.BaseURL == "" {
return fmt.Errorf("base_url is required")
}
if c.UIBaseURL == "" {
c.UIBaseURL = c.BaseURL
} else if err := validateAbsoluteURL("ui_base_url", c.UIBaseURL); err != nil {
return err
}
if c.Storage.URL == "" && c.Storage.Path == "" {
return fmt.Errorf("storage.url or storage.path is required")
2026-01-20 21:52:44 +00:00
}
switch c.Database.Driver {
case "sqlite":
if c.Database.Path == "" {
return fmt.Errorf("database.path is required for sqlite driver")
}
case "postgres":
if c.Database.URL == "" {
return fmt.Errorf("database.url is required for postgres driver")
}
default:
return fmt.Errorf("invalid database.driver %q (must be sqlite or postgres)", c.Database.Driver)
2026-01-20 21:52:44 +00:00
}
// Validate log level
switch strings.ToLower(c.Log.Level) {
case "debug", "info", "warn", "error":
// OK
default:
return fmt.Errorf("invalid log level %q (must be debug, info, warn, or error)", c.Log.Level)
}
// Validate log format
switch strings.ToLower(c.Log.Format) {
case "text", "json":
// OK
default:
return fmt.Errorf("invalid log format %q (must be text or json)", c.Log.Format)
}
// Validate max size if specified
if c.Storage.MaxSize != "" {
if _, err := ParseSize(c.Storage.MaxSize); err != nil {
return fmt.Errorf("invalid storage.max_size: %w", err)
}
}
// Validate direct serve TTL if specified
if c.Storage.DirectServeTTL != "" {
if _, err := time.ParseDuration(c.Storage.DirectServeTTL); err != nil {
return fmt.Errorf("invalid storage.direct_serve_ttl %q: %w", c.Storage.DirectServeTTL, err)
}
}
// Validate direct serve base URL if specified
if c.Storage.DirectServeBaseURL != "" {
if err := validateAbsoluteURL("storage.direct_serve_base_url", c.Storage.DirectServeBaseURL); err != nil {
return err
}
}
Add storage.cache_artifacts to serve artifacts without storing them (#387) * Add storage.passthrough to serve artifacts without storing them With storage.passthrough enabled the proxy streams every artifact from upstream to the client and never writes it to storage or the cache database. Metadata filtering, cooldown and the denylist work as before, so the proxy can sit behind another cache (e.g. an Artifactory remote) purely as a policy layer without holding a second copy of every package. Artifacts whose digest is known up front (OCI, Swift, Helm) are verified while streaming. Their responses are sent chunked and the connection is aborted on a mismatch, so a client never receives a tampered artifact as a complete response. Passthrough is rejected together with scanning, direct_serve and mirror_api, and the mirror command refuses to run with it, since all of them depend on stored artifacts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Address review: abort truncated streams, guard every cache read - serveArtifact aborts the response on any body read error, and when fewer bytes than the declared size were written, not only on a digest mismatch. A failed or short upstream body in passthrough mode no longer reaches the client as a complete 200. - Streamed upstream read failures are logged and counted as stream_failed upstream errors before the response is aborted. - checkCache reports a miss when artifact caching is off, so every cache read path, including Swift archive HEAD requests, ignores entries stored before the mode was enabled. Denylisted versions are still rejected. - Rename storage.passthrough to storage.cache_artifacts (default true, PROXY_STORAGE_CACHE_ARTIFACTS), which names what actually changes. Tests cover an OCI blob shorter than its Content-Length and an npm tarball whose chunked response ends without the final chunk, both served through the HTTP handlers, plus a Swift HEAD request against a cached archive. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Rename Proxy.Passthrough to StreamArtifacts Match the storage.cache_artifacts option: the field says what changes (artifacts are streamed instead of stored) and its zero value keeps the usual caching behaviour for every Proxy built without the server config, such as the mirror command and tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 15:28:20 +07:00
if err := c.validateCacheArtifacts(); err != nil {
return err
}
// Validate metadata TTL if specified
if c.MetadataTTL != "" && c.MetadataTTL != "0" {
if _, err := time.ParseDuration(c.MetadataTTL); err != nil {
return fmt.Errorf("invalid metadata_ttl %q: %w", c.MetadataTTL, err)
}
}
if err := c.validateDownloadSizeLimits(); err != nil {
return err
}
if err := validateHTTPTimeout(c.HTTPTimeout); err != nil {
return err
}
if err := validateHitFlushInterval(c.Database.HitFlushInterval); err != nil {
return err
}
return c.validateComponents()
}
Add storage.cache_artifacts to serve artifacts without storing them (#387) * Add storage.passthrough to serve artifacts without storing them With storage.passthrough enabled the proxy streams every artifact from upstream to the client and never writes it to storage or the cache database. Metadata filtering, cooldown and the denylist work as before, so the proxy can sit behind another cache (e.g. an Artifactory remote) purely as a policy layer without holding a second copy of every package. Artifacts whose digest is known up front (OCI, Swift, Helm) are verified while streaming. Their responses are sent chunked and the connection is aborted on a mismatch, so a client never receives a tampered artifact as a complete response. Passthrough is rejected together with scanning, direct_serve and mirror_api, and the mirror command refuses to run with it, since all of them depend on stored artifacts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Address review: abort truncated streams, guard every cache read - serveArtifact aborts the response on any body read error, and when fewer bytes than the declared size were written, not only on a digest mismatch. A failed or short upstream body in passthrough mode no longer reaches the client as a complete 200. - Streamed upstream read failures are logged and counted as stream_failed upstream errors before the response is aborted. - checkCache reports a miss when artifact caching is off, so every cache read path, including Swift archive HEAD requests, ignores entries stored before the mode was enabled. Denylisted versions are still rejected. - Rename storage.passthrough to storage.cache_artifacts (default true, PROXY_STORAGE_CACHE_ARTIFACTS), which names what actually changes. Tests cover an OCI blob shorter than its Content-Length and an npm tarball whose chunked response ends without the final chunk, both served through the HTTP handlers, plus a Swift HEAD request against a cached archive. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Rename Proxy.Passthrough to StreamArtifacts Match the storage.cache_artifacts option: the field says what changes (artifacts are streamed instead of stored) and its zero value keeps the usual caching behaviour for every Proxy built without the server config, such as the mirror command and tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 15:28:20 +07:00
func (c *Config) validateCacheArtifacts() error {
if c.Storage.CacheArtifacts {
return nil
}
switch {
case c.Scanning.Enabled:
return fmt.Errorf("storage.cache_artifacts: false cannot be combined with scanning.enabled: scanning needs stored artifacts")
case c.Storage.DirectServe:
return fmt.Errorf("storage.cache_artifacts: false cannot be combined with storage.direct_serve: no artifacts are stored to redirect to")
case c.MirrorAPI:
return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served")
case c.Hosted.Enabled:
return fmt.Errorf("storage.cache_artifacts: false cannot be combined with hosted.enabled: pushed packages would never be served")
Add storage.cache_artifacts to serve artifacts without storing them (#387) * Add storage.passthrough to serve artifacts without storing them With storage.passthrough enabled the proxy streams every artifact from upstream to the client and never writes it to storage or the cache database. Metadata filtering, cooldown and the denylist work as before, so the proxy can sit behind another cache (e.g. an Artifactory remote) purely as a policy layer without holding a second copy of every package. Artifacts whose digest is known up front (OCI, Swift, Helm) are verified while streaming. Their responses are sent chunked and the connection is aborted on a mismatch, so a client never receives a tampered artifact as a complete response. Passthrough is rejected together with scanning, direct_serve and mirror_api, and the mirror command refuses to run with it, since all of them depend on stored artifacts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Address review: abort truncated streams, guard every cache read - serveArtifact aborts the response on any body read error, and when fewer bytes than the declared size were written, not only on a digest mismatch. A failed or short upstream body in passthrough mode no longer reaches the client as a complete 200. - Streamed upstream read failures are logged and counted as stream_failed upstream errors before the response is aborted. - checkCache reports a miss when artifact caching is off, so every cache read path, including Swift archive HEAD requests, ignores entries stored before the mode was enabled. Denylisted versions are still rejected. - Rename storage.passthrough to storage.cache_artifacts (default true, PROXY_STORAGE_CACHE_ARTIFACTS), which names what actually changes. Tests cover an OCI blob shorter than its Content-Length and an npm tarball whose chunked response ends without the final chunk, both served through the HTTP handlers, plus a Swift HEAD request against a cached archive. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Rename Proxy.Passthrough to StreamArtifacts Match the storage.cache_artifacts option: the field says what changes (artifacts are streamed instead of stored) and its zero value keeps the usual caching behaviour for every Proxy built without the server config, such as the mirror command and tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 15:28:20 +07:00
}
return nil
}
func (c *Config) validateComponents() error {
Cache rewritten npm and Composer metadata (#402) The npm and Composer handlers rewrite every metadata document they serve so that download URLs point at the proxy: decode the whole document into generic maps, change the URLs, encode it again, and for Composer expand the minified format first. That ran on every request, cached metadata included. With metadata caching on and upstream out of the picture, a cached request still cost 7 ms for @babel/core, 17 ms for symfony/console and 155 ms and 134 MB of allocations for typescript, and an 8 vCPU VM serving cached npm packuments to 20 clients ran the proxy at 450% CPU for about 1,100 requests a second. Rewritten documents are now kept in memory, keyed by the ecosystem, proxy URL, package and a SHA-256 of the raw document, so new bytes from upstream are rewritten again and nothing is served stale. Requests that arrive while a document is being rewritten wait for that rewrite rather than running their own; a waiter leaves when its client does, and the rewrite still completes and is cached. The denylist is fixed at startup, so it needs no place in the key. Cooldown filtering depends on the current time, so with cooldown on the cache is bypassed. metadata_rewrite_cache_size bounds the cache (default "256MB", least recently used out first, "0" to rewrite on every request). NewProxy callers keep the old behaviour unless they set it. Whole cached requests, measured locally: @babel/core 7.1 ms -> 0.38 ms, 54,489 -> 119 allocations typescript 155 ms -> 10.5 ms, 1.37M -> 133 allocations symfony/console 17.4 ms -> 0.48 ms, 165,909 -> 123 allocations What remains is reading the raw document from storage and hashing it.
2026-10-03 04:09:15 -07:00
if err := validateMetadataRewriteCacheSize(c.MetadataRewriteCacheSize); err != nil {
return err
}
Add age-based retention for cached artifacts (#428) * Add age-based retention for cached artifacts Cached artifacts stayed forever unless storage.max_size pushed them out. storage.retention adds a default, per-ecosystem and per-package duration after which an artifact nobody downloaded gets evicted. Ecosystems opt in one at a time: a handler registers with retention.Default, and until it does, configuring that ecosystem (or one of its packages) fails validation, and the default doesn't touch it. This commit registers none, so nothing changes yet. Startup logs which ecosystems the default covers and which it doesn't. An artifact counts as expired when both its fetch time and its last access are older than the cutoff. Checking only the last access would evict a refetched artifact again right away, since clearing a record keeps its old access time and a cache miss doesn't record a hit. The sweep scans artifacts in id windows, so no single query holds the SQLite connection for long, and checks each row against its own rule. Expired records are cleared and their files go through pending_deletes instead of being deleted directly, because a buffered hit can make an artifact that's being served right now look expired. Buffered hits are flushed before each sweep, and one sweep clears at most 10,000 artifacts. Since a big first sweep can queue a lot, reclaim now keeps working through due entries for up to 30 seconds per tick instead of stopping after 100, and pending_deletes gets an index on queued_at. Also: - proxy_artifacts_evicted_total{reason, ecosystem} counts LRU and retention evictions. - gradle.build_cache.max_age accepts "7d", as its comment always said. Refs #306 * Document storage retention Describes the new storage.retention block, the order rules apply in, that the default only covers ecosystems that support retention yet, and what to expect from the first sweep over an old cache. Also lists the new eviction counter and notes that gradle max_age takes days. Refs #306 * Fix retention timing on Postgres and tighten the sweep On Postgres, fetched_at and last_accessed_at are TIMESTAMP columns without a zone. They hold the local time the proxy wrote, but lib/pq reads them back labelled UTC. East of UTC that makes every artifact look hours younger than it is, so the sweep evicted late: two hours in Berlin, nine in Tokyo. The candidates' times are now read back as local time, and the database tests run on both SQLite and Postgres, with one that sets the local zone to Asia/Tokyo. A few smaller things: - When a delete fails, reclaim stops for the rest of the minute instead of trying every queued path. Otherwise a storage outage turns into thirty seconds of failed deletes and warnings every minute. - If the buffered hits can't be written, the sweep skips that round. Running on stale access times could clear something that is being downloaded right now. - DefaultCanonical only accepts a package key when rebuilding it gives the same PURL. For alpine and deb it would otherwise produce keys that never match. - sweep_interval has to be at least a minute. - New tests cover the eviction counter and an artifact that gets a hit between the scan and the clear. - The docs no longer say nothing changes without retention, and they point out that the example's ecosystem and package rules only pass validation once those ecosystems support retention. Refs #306 * Reject qualifiers and subpaths in retention package keys A rule names a whole package. Qualifiers like ?type=pom were dropped on the way to the stored PURL and the rule quietly covered every file of the package. Now such a key fails validation. Refs #306 * Say when a package key doesn't match how packages are stored A package rule for an ecosystem that supports retention, written in a form its packages aren't stored under (pkg:npm/babel/core without the @), failed with "retention is not supported for npm packages". Once the ecosystem is enabled that is no longer true. Such a key now fails with "does not match how npm packages are stored" and points to the package rule forms in the docs. Keys for ecosystems without retention support keep the old message. Refs #306 * Note that re-running the mirror keeps packages cached A mirror run over a package that is still cached goes through the normal cache lookup and records a download, so the package's age starts over. The docs only said that mirrored packages age from the time they were mirrored. Refs #306 * Say when an ecosystem takes no package rules Some ecosystems will support retention without package rules, because nothing in their stored records names a package the way a PURL would (conan, helm). A package key for one of them used to fail with "does not match how conan packages are stored; see the package rule forms", which sends people looking for a form that doesn't exist. It now says package rules aren't supported there and points to the ecosystem rule. Refs #306 * Use helm as the example of an ecosystem without package rules Conan gets package rules once its handler stores archives under the recipe name, so the config test that shows the refusal now uses helm, whose cache records name the repository, not the chart. Refs #306 * Say that each cached file ages on its own The retention docs said each artifact, each version of a package, ages on its own. A version can have several cached files (a jar and its pom, several wheels, Conan sources and binaries), and each of them ages separately. Refs #306 * Clear an expired artifact and queue its file in one transaction The sweep cleared the record and queued the file for deletion as two separate writes. If queueing failed, or the process stopped in between, the file stayed in storage with no record pointing at it and no queue entry, and no later sweep would find it again. Both writes now share a transaction, so a failed queue rolls the clear back and the next sweep retries. Refs #306 * Show artifact evictions on the analytics page proxy_artifacts_evicted_total had no place on /ui/analytics. The Runtime card now lists evicted artifacts by reason and ecosystem, without the failure tint, and the coverage test records an eviction so it doesn't depend on test order. Refs #306 * Store artifact times in UTC fetched_at and last_accessed_at are now written in UTC, and the retention cutoffs are compared in UTC, the same convention #434 uses for the metadata cache. lib/pq sends a time with its offset and a Postgres TIMESTAMP keeps only the wall clock, so UTC on write is what reads back correctly. That replaces reinterpreting the times as local on read. Rows written before this still hold local time. East of UTC they look younger by the offset until they are rewritten, so retention evicts them a few hours late once; west of UTC that much early. Refs #306 --------- Co-authored-by: t <t@t>
2026-10-12 10:06:01 +02:00
if _, err := c.RetentionRules(retention.Default); err != nil {
return err
}
if _, err := denylist.New(c.Denylist.Packages); err != nil {
return err
}
if err := c.validateUpstream(); err != nil {
Add storage backend probe to /health (closes #73) (#119) * config: add Health.StorageProbeInterval * metrics: add proxy_health_probe_failures_total counter * server: add storageProbe with happy-path test * server: add storageProbe failure-mode tests * server: add healthCache with TTL, single-flight, transition logging * server: wire storage probe into /health * server: update TestHealthEndpoint for JSON; wire healthCache into newTestServer Also fix Windows file-locking issue in storageProbe: close the reader explicitly before Delete so the file handle is released prior to os.Remove. * server: clean up stale comment in storageProbe * docs: document storage health probe and new metric * docs: regenerate Swagger for /health JSON response * server: simplify rc.Close error handling in storageProbe * server: defer probe cleanup so size/open/read/verify failures don't leak objects Previously, storageProbe only called Delete on the success path. Any failure between Store and the final Delete (size mismatch, Open error, mid-stream read failure, content mismatch) left the probe object orphaned in the storage backend. With caching disabled and Kubernetes-rate probing, the leak could accumulate noticeably on backends like S3. Use a named return + defer to attempt Delete after every successful Store. The earlier-step failure remains the primary error; Delete failure only surfaces as step="delete" when nothing else went wrong. Add a table-driven test that asserts cleanup runs for each non-delete failure path. Reported by Copilot on #119. * config: validate health.storage_probe_interval in Config.Validate The new duration field was only validated at use time in newHealthCache. The existing codebase already validates other duration fields (MetadataTTL, DirectServeTTL, Gradle.MaxAge, Gradle.SweepInterval) in Config.Validate() so misconfiguration fails fast at startup with a config-key-specific error. Match that pattern. The parse-at-use code in newHealthCache stays as a safety net, mirroring the MetadataTTL precedent. Reported by Copilot on #119. * docs: lowercase "counter" in metrics table for consistency Other rows in the table use lowercase type names (counter/gauge/histogram). Match that style. Reported by Copilot on #119. * docs: include size-check step in /health probe description The probe is write → size-check → read → verify → delete; the architecture note was missing the size-check step. Reported by Copilot on #119. * server: address andrew's review on #119 - Drop unused callerCtx parameter from healthCache.Check (Check is now parameter-less; the comment-only "accepted for symmetry" justification wasn't carrying its weight). - Emit "storage": {"status": "skipped"} on DB short-circuit instead of omitting the key, so monitors expecting a fixed key set keep working. - Reject negative storage_probe_interval at config validation time (previously parsed and silently behaved like "0"). - Extract HealthConfig.Validate to keep Config.Validate under the gocognit threshold and match the existing GradleBuildCacheConfig pattern. - README Health Check section: note that /health is intended as a readiness probe rather than a liveness probe (Check holds a mutex for up to the 10s probe timeout). - cmd/proxy/main.go godoc: column-align the new env var with the surrounding Gradle entries. Reported by andrew on #119.
2026-05-22 14:14:01 +03:00
return err
}
if err := c.Health.Validate(); err != nil {
return err
}
Add pre-cache artifact scanning hook (trivy/ClamAV/Wiz/custom) (#298) Runs fetched artifacts through pluggable external scanners after they're staged in storage but before they're committed to the cache DB, so a block verdict deletes the object instead of ever exposing it to a client. Scanners pull the staged bytes themselves via a short-lived HMAC-signed internal route rather than the proxy pushing bytes to them, keeping the mechanism storage-backend-agnostic and avoiding uploading potentially huge artifacts through the proxy's own egress. Hardening baked in from the start: the internal scan-fetch route is gated both at router-mount time and in the handler so it's inert whenever scanning is disabled or unsigned; the signing key is mandatory whenever scanning is enabled, enforced directly in scanner.NewGroup rather than relying on callers to invoke config validation; the scan call and the delete-on-block cleanup both run on a context detached from the client's, so a client disconnecting mid-scan can't be mistaken for a scanner failure, doesn't cause a legitimate artifact to be deleted, and doesn't leave a genuinely blocked artifact's bytes orphaned in storage; and scanner infrastructure errors (connection failures, internal hostnames) are never forwarded verbatim to anonymous clients, only a generic message. The scan-error metric also correctly distinguishes a scanner's own timeout from being cancelled because a sibling scanner already decided the verdict. Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-03 05:06:44 -04:00
if err := c.Scanning.Validate(); err != nil {
return err
}
if err := c.Hosted.Validate(); err != nil {
return err
}
return c.Gradle.BuildCache.Validate()
}
Add storage backend probe to /health (closes #73) (#119) * config: add Health.StorageProbeInterval * metrics: add proxy_health_probe_failures_total counter * server: add storageProbe with happy-path test * server: add storageProbe failure-mode tests * server: add healthCache with TTL, single-flight, transition logging * server: wire storage probe into /health * server: update TestHealthEndpoint for JSON; wire healthCache into newTestServer Also fix Windows file-locking issue in storageProbe: close the reader explicitly before Delete so the file handle is released prior to os.Remove. * server: clean up stale comment in storageProbe * docs: document storage health probe and new metric * docs: regenerate Swagger for /health JSON response * server: simplify rc.Close error handling in storageProbe * server: defer probe cleanup so size/open/read/verify failures don't leak objects Previously, storageProbe only called Delete on the success path. Any failure between Store and the final Delete (size mismatch, Open error, mid-stream read failure, content mismatch) left the probe object orphaned in the storage backend. With caching disabled and Kubernetes-rate probing, the leak could accumulate noticeably on backends like S3. Use a named return + defer to attempt Delete after every successful Store. The earlier-step failure remains the primary error; Delete failure only surfaces as step="delete" when nothing else went wrong. Add a table-driven test that asserts cleanup runs for each non-delete failure path. Reported by Copilot on #119. * config: validate health.storage_probe_interval in Config.Validate The new duration field was only validated at use time in newHealthCache. The existing codebase already validates other duration fields (MetadataTTL, DirectServeTTL, Gradle.MaxAge, Gradle.SweepInterval) in Config.Validate() so misconfiguration fails fast at startup with a config-key-specific error. Match that pattern. The parse-at-use code in newHealthCache stays as a safety net, mirroring the MetadataTTL precedent. Reported by Copilot on #119. * docs: lowercase "counter" in metrics table for consistency Other rows in the table use lowercase type names (counter/gauge/histogram). Match that style. Reported by Copilot on #119. * docs: include size-check step in /health probe description The probe is write → size-check → read → verify → delete; the architecture note was missing the size-check step. Reported by Copilot on #119. * server: address andrew's review on #119 - Drop unused callerCtx parameter from healthCache.Check (Check is now parameter-less; the comment-only "accepted for symmetry" justification wasn't carrying its weight). - Emit "storage": {"status": "skipped"} on DB short-circuit instead of omitting the key, so monitors expecting a fixed key set keep working. - Reject negative storage_probe_interval at config validation time (previously parsed and silently behaved like "0"). - Extract HealthConfig.Validate to keep Config.Validate under the gocognit threshold and match the existing GradleBuildCacheConfig pattern. - README Health Check section: note that /health is intended as a readiness probe rather than a liveness probe (Check holds a mutex for up to the 10s probe timeout). - cmd/proxy/main.go godoc: column-align the new env var with the surrounding Gradle entries. Reported by andrew on #119.
2026-05-22 14:14:01 +03:00
// Validate checks the /health configuration. An unset interval is allowed
// (the cache uses its default); explicit values must parse and be non-negative.
func (h *HealthConfig) Validate() error {
if h.StorageProbeInterval == "" || h.StorageProbeInterval == "0" {
return nil
}
d, err := time.ParseDuration(h.StorageProbeInterval)
if err != nil {
return fmt.Errorf("invalid health.storage_probe_interval %q: %w", h.StorageProbeInterval, err)
}
if d < 0 {
return fmt.Errorf("invalid health.storage_probe_interval %q: must be non-negative", h.StorageProbeInterval)
}
return nil
}
// Validate checks Gradle build cache settings, applying the default upload
// size if unset.
func (g *GradleBuildCacheConfig) Validate() error {
if g.MaxUploadSize == "" {
g.MaxUploadSize = defaultGradleMaxUploadSizeStr
}
uploadSize, err := ParseSize(g.MaxUploadSize)
if err != nil {
return fmt.Errorf("invalid gradle.build_cache.max_upload_size: %w", err)
}
if uploadSize <= 0 {
return fmt.Errorf("invalid gradle.build_cache.max_upload_size %q: must be > 0", g.MaxUploadSize)
}
if g.MaxAge != "" && g.MaxAge != "0" {
Add age-based retention for cached artifacts (#428) * Add age-based retention for cached artifacts Cached artifacts stayed forever unless storage.max_size pushed them out. storage.retention adds a default, per-ecosystem and per-package duration after which an artifact nobody downloaded gets evicted. Ecosystems opt in one at a time: a handler registers with retention.Default, and until it does, configuring that ecosystem (or one of its packages) fails validation, and the default doesn't touch it. This commit registers none, so nothing changes yet. Startup logs which ecosystems the default covers and which it doesn't. An artifact counts as expired when both its fetch time and its last access are older than the cutoff. Checking only the last access would evict a refetched artifact again right away, since clearing a record keeps its old access time and a cache miss doesn't record a hit. The sweep scans artifacts in id windows, so no single query holds the SQLite connection for long, and checks each row against its own rule. Expired records are cleared and their files go through pending_deletes instead of being deleted directly, because a buffered hit can make an artifact that's being served right now look expired. Buffered hits are flushed before each sweep, and one sweep clears at most 10,000 artifacts. Since a big first sweep can queue a lot, reclaim now keeps working through due entries for up to 30 seconds per tick instead of stopping after 100, and pending_deletes gets an index on queued_at. Also: - proxy_artifacts_evicted_total{reason, ecosystem} counts LRU and retention evictions. - gradle.build_cache.max_age accepts "7d", as its comment always said. Refs #306 * Document storage retention Describes the new storage.retention block, the order rules apply in, that the default only covers ecosystems that support retention yet, and what to expect from the first sweep over an old cache. Also lists the new eviction counter and notes that gradle max_age takes days. Refs #306 * Fix retention timing on Postgres and tighten the sweep On Postgres, fetched_at and last_accessed_at are TIMESTAMP columns without a zone. They hold the local time the proxy wrote, but lib/pq reads them back labelled UTC. East of UTC that makes every artifact look hours younger than it is, so the sweep evicted late: two hours in Berlin, nine in Tokyo. The candidates' times are now read back as local time, and the database tests run on both SQLite and Postgres, with one that sets the local zone to Asia/Tokyo. A few smaller things: - When a delete fails, reclaim stops for the rest of the minute instead of trying every queued path. Otherwise a storage outage turns into thirty seconds of failed deletes and warnings every minute. - If the buffered hits can't be written, the sweep skips that round. Running on stale access times could clear something that is being downloaded right now. - DefaultCanonical only accepts a package key when rebuilding it gives the same PURL. For alpine and deb it would otherwise produce keys that never match. - sweep_interval has to be at least a minute. - New tests cover the eviction counter and an artifact that gets a hit between the scan and the clear. - The docs no longer say nothing changes without retention, and they point out that the example's ecosystem and package rules only pass validation once those ecosystems support retention. Refs #306 * Reject qualifiers and subpaths in retention package keys A rule names a whole package. Qualifiers like ?type=pom were dropped on the way to the stored PURL and the rule quietly covered every file of the package. Now such a key fails validation. Refs #306 * Say when a package key doesn't match how packages are stored A package rule for an ecosystem that supports retention, written in a form its packages aren't stored under (pkg:npm/babel/core without the @), failed with "retention is not supported for npm packages". Once the ecosystem is enabled that is no longer true. Such a key now fails with "does not match how npm packages are stored" and points to the package rule forms in the docs. Keys for ecosystems without retention support keep the old message. Refs #306 * Note that re-running the mirror keeps packages cached A mirror run over a package that is still cached goes through the normal cache lookup and records a download, so the package's age starts over. The docs only said that mirrored packages age from the time they were mirrored. Refs #306 * Say when an ecosystem takes no package rules Some ecosystems will support retention without package rules, because nothing in their stored records names a package the way a PURL would (conan, helm). A package key for one of them used to fail with "does not match how conan packages are stored; see the package rule forms", which sends people looking for a form that doesn't exist. It now says package rules aren't supported there and points to the ecosystem rule. Refs #306 * Use helm as the example of an ecosystem without package rules Conan gets package rules once its handler stores archives under the recipe name, so the config test that shows the refusal now uses helm, whose cache records name the repository, not the chart. Refs #306 * Say that each cached file ages on its own The retention docs said each artifact, each version of a package, ages on its own. A version can have several cached files (a jar and its pom, several wheels, Conan sources and binaries), and each of them ages separately. Refs #306 * Clear an expired artifact and queue its file in one transaction The sweep cleared the record and queued the file for deletion as two separate writes. If queueing failed, or the process stopped in between, the file stayed in storage with no record pointing at it and no queue entry, and no later sweep would find it again. Both writes now share a transaction, so a failed queue rolls the clear back and the next sweep retries. Refs #306 * Show artifact evictions on the analytics page proxy_artifacts_evicted_total had no place on /ui/analytics. The Runtime card now lists evicted artifacts by reason and ecosystem, without the failure tint, and the coverage test records an eviction so it doesn't depend on test order. Refs #306 * Store artifact times in UTC fetched_at and last_accessed_at are now written in UTC, and the retention cutoffs are compared in UTC, the same convention #434 uses for the metadata cache. lib/pq sends a time with its offset and a Postgres TIMESTAMP keeps only the wall clock, so UTC on write is what reads back correctly. That replaces reinterpreting the times as local on read. Rows written before this still hold local time. East of UTC they look younger by the offset until they are rewritten, so retention evicts them a few hours late once; west of UTC that much early. Refs #306 --------- Co-authored-by: t <t@t>
2026-10-12 10:06:01 +02:00
if _, err := parseDayDuration(g.MaxAge); err != nil {
return fmt.Errorf("invalid gradle.build_cache.max_age %q: %w", g.MaxAge, err)
}
}
if g.MaxSize != "" {
if _, err := ParseSize(g.MaxSize); err != nil {
return fmt.Errorf("invalid gradle.build_cache.max_size: %w", err)
}
}
if g.SweepInterval != "" {
d, err := time.ParseDuration(g.SweepInterval)
if err != nil {
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: %w", g.SweepInterval, err)
}
if d <= 0 {
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: must be > 0", g.SweepInterval)
}
}
2026-01-20 21:52:44 +00:00
return nil
}
const (
defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default
defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default
defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default
defaultHitFlushInterval = time.Second
defaultMetadataMaxSize = 100 << 20
Cache rewritten npm and Composer metadata (#402) The npm and Composer handlers rewrite every metadata document they serve so that download URLs point at the proxy: decode the whole document into generic maps, change the URLs, encode it again, and for Composer expand the minified format first. That ran on every request, cached metadata included. With metadata caching on and upstream out of the picture, a cached request still cost 7 ms for @babel/core, 17 ms for symfony/console and 155 ms and 134 MB of allocations for typescript, and an 8 vCPU VM serving cached npm packuments to 20 clients ran the proxy at 450% CPU for about 1,100 requests a second. Rewritten documents are now kept in memory, keyed by the ecosystem, proxy URL, package and a SHA-256 of the raw document, so new bytes from upstream are rewritten again and nothing is served stale. Requests that arrive while a document is being rewritten wait for that rewrite rather than running their own; a waiter leaves when its client does, and the rewrite still completes and is cached. The denylist is fixed at startup, so it needs no place in the key. Cooldown filtering depends on the current time, so with cooldown on the cache is bypassed. metadata_rewrite_cache_size bounds the cache (default "256MB", least recently used out first, "0" to rewrite on every request). NewProxy callers keep the old behaviour unless they set it. Whole cached requests, measured locally: @babel/core 7.1 ms -> 0.38 ms, 54,489 -> 119 allocations typescript 155 ms -> 10.5 ms, 1.37M -> 133 allocations symfony/console 17.4 ms -> 0.48 ms, 165,909 -> 123 allocations What remains is reading the raw document from storage and hashing it.
2026-10-03 04:09:15 -07:00
defaultMetadataRewriteCacheSize = 256 << 20
defaultGradleBuildCacheMaxUploadSize = 100 << 20
defaultGradleBuildCacheSweepInterval = 10 * time.Minute
defaultGradleMaxUploadSizeStr = "100MB"
defaultGradleSweepIntervalStr = "10m"
Add age-based retention for cached artifacts (#428) * Add age-based retention for cached artifacts Cached artifacts stayed forever unless storage.max_size pushed them out. storage.retention adds a default, per-ecosystem and per-package duration after which an artifact nobody downloaded gets evicted. Ecosystems opt in one at a time: a handler registers with retention.Default, and until it does, configuring that ecosystem (or one of its packages) fails validation, and the default doesn't touch it. This commit registers none, so nothing changes yet. Startup logs which ecosystems the default covers and which it doesn't. An artifact counts as expired when both its fetch time and its last access are older than the cutoff. Checking only the last access would evict a refetched artifact again right away, since clearing a record keeps its old access time and a cache miss doesn't record a hit. The sweep scans artifacts in id windows, so no single query holds the SQLite connection for long, and checks each row against its own rule. Expired records are cleared and their files go through pending_deletes instead of being deleted directly, because a buffered hit can make an artifact that's being served right now look expired. Buffered hits are flushed before each sweep, and one sweep clears at most 10,000 artifacts. Since a big first sweep can queue a lot, reclaim now keeps working through due entries for up to 30 seconds per tick instead of stopping after 100, and pending_deletes gets an index on queued_at. Also: - proxy_artifacts_evicted_total{reason, ecosystem} counts LRU and retention evictions. - gradle.build_cache.max_age accepts "7d", as its comment always said. Refs #306 * Document storage retention Describes the new storage.retention block, the order rules apply in, that the default only covers ecosystems that support retention yet, and what to expect from the first sweep over an old cache. Also lists the new eviction counter and notes that gradle max_age takes days. Refs #306 * Fix retention timing on Postgres and tighten the sweep On Postgres, fetched_at and last_accessed_at are TIMESTAMP columns without a zone. They hold the local time the proxy wrote, but lib/pq reads them back labelled UTC. East of UTC that makes every artifact look hours younger than it is, so the sweep evicted late: two hours in Berlin, nine in Tokyo. The candidates' times are now read back as local time, and the database tests run on both SQLite and Postgres, with one that sets the local zone to Asia/Tokyo. A few smaller things: - When a delete fails, reclaim stops for the rest of the minute instead of trying every queued path. Otherwise a storage outage turns into thirty seconds of failed deletes and warnings every minute. - If the buffered hits can't be written, the sweep skips that round. Running on stale access times could clear something that is being downloaded right now. - DefaultCanonical only accepts a package key when rebuilding it gives the same PURL. For alpine and deb it would otherwise produce keys that never match. - sweep_interval has to be at least a minute. - New tests cover the eviction counter and an artifact that gets a hit between the scan and the clear. - The docs no longer say nothing changes without retention, and they point out that the example's ecosystem and package rules only pass validation once those ecosystems support retention. Refs #306 * Reject qualifiers and subpaths in retention package keys A rule names a whole package. Qualifiers like ?type=pom were dropped on the way to the stored PURL and the rule quietly covered every file of the package. Now such a key fails validation. Refs #306 * Say when a package key doesn't match how packages are stored A package rule for an ecosystem that supports retention, written in a form its packages aren't stored under (pkg:npm/babel/core without the @), failed with "retention is not supported for npm packages". Once the ecosystem is enabled that is no longer true. Such a key now fails with "does not match how npm packages are stored" and points to the package rule forms in the docs. Keys for ecosystems without retention support keep the old message. Refs #306 * Note that re-running the mirror keeps packages cached A mirror run over a package that is still cached goes through the normal cache lookup and records a download, so the package's age starts over. The docs only said that mirrored packages age from the time they were mirrored. Refs #306 * Say when an ecosystem takes no package rules Some ecosystems will support retention without package rules, because nothing in their stored records names a package the way a PURL would (conan, helm). A package key for one of them used to fail with "does not match how conan packages are stored; see the package rule forms", which sends people looking for a form that doesn't exist. It now says package rules aren't supported there and points to the ecosystem rule. Refs #306 * Use helm as the example of an ecosystem without package rules Conan gets package rules once its handler stores archives under the recipe name, so the config test that shows the refusal now uses helm, whose cache records name the repository, not the chart. Refs #306 * Say that each cached file ages on its own The retention docs said each artifact, each version of a package, ages on its own. A version can have several cached files (a jar and its pom, several wheels, Conan sources and binaries), and each of them ages separately. Refs #306 * Clear an expired artifact and queue its file in one transaction The sweep cleared the record and queued the file for deletion as two separate writes. If queueing failed, or the process stopped in between, the file stayed in storage with no record pointing at it and no queue entry, and no later sweep would find it again. Both writes now share a transaction, so a failed queue rolls the clear back and the next sweep retries. Refs #306 * Show artifact evictions on the analytics page proxy_artifacts_evicted_total had no place on /ui/analytics. The Runtime card now lists evicted artifacts by reason and ecosystem, without the failure tint, and the coverage test records an eviction so it doesn't depend on test order. Refs #306 * Store artifact times in UTC fetched_at and last_accessed_at are now written in UTC, and the retention cutoffs are compared in UTC, the same convention #434 uses for the metadata cache. lib/pq sends a time with its offset and a Postgres TIMESTAMP keeps only the wall clock, so UTC on write is what reads back correctly. That replaces reinterpreting the times as local on read. Rows written before this still hold local time. East of UTC they look younger by the offset until they are rewritten, so retention evicts them a few hours late once; west of UTC that much early. Refs #306 --------- Co-authored-by: t <t@t>
2026-10-12 10:06:01 +02:00
defaultRetentionSweepInterval = 10 * time.Minute
defaultRetentionSweepIntervalStr = "10m"
minRetentionSweepInterval = time.Minute
maxDurationDays = 36500
Add pre-cache artifact scanning hook (trivy/ClamAV/Wiz/custom) (#298) Runs fetched artifacts through pluggable external scanners after they're staged in storage but before they're committed to the cache DB, so a block verdict deletes the object instead of ever exposing it to a client. Scanners pull the staged bytes themselves via a short-lived HMAC-signed internal route rather than the proxy pushing bytes to them, keeping the mechanism storage-backend-agnostic and avoiding uploading potentially huge artifacts through the proxy's own egress. Hardening baked in from the start: the internal scan-fetch route is gated both at router-mount time and in the handler so it's inert whenever scanning is disabled or unsigned; the signing key is mandatory whenever scanning is enabled, enforced directly in scanner.NewGroup rather than relying on callers to invoke config validation; the scan call and the delete-on-block cleanup both run on a context detached from the client's, so a client disconnecting mid-scan can't be mistaken for a scanner failure, doesn't cause a legitimate artifact to be deleted, and doesn't leave a genuinely blocked artifact's bytes orphaned in storage; and scanner infrastructure errors (connection failures, internal hostnames) are never forwarded verbatim to anonymous clients, only a generic message. The scan-error metric also correctly distinguishes a scanner's own timeout from being cancelled because a sibling scanner already decided the verdict. Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-03 05:06:44 -04:00
defaultScanningTimeoutStr = "30s"
)
// ParseMaxSize returns the maximum cache size in bytes.
// Returns 0 if unset or explicitly disabled (meaning unlimited).
func (c *Config) ParseMaxSize() int64 {
if c.Storage.MaxSize == "" || c.Storage.MaxSize == "0" {
return 0
}
size, err := ParseSize(c.Storage.MaxSize)
if err != nil {
return 0
}
return size
}
func validateHTTPTimeout(s string) error {
if s == "" || s == "0" {
return nil
}
d, err := time.ParseDuration(s)
if err != nil {
return fmt.Errorf("invalid http_timeout %q: %w", s, err)
}
if d < 0 {
return fmt.Errorf("invalid http_timeout %q: must be non-negative", s)
}
return nil
}
func validateMetadataMaxSize(s string) error {
if s == "" {
return nil
}
size, err := ParseSize(s)
if err != nil {
return fmt.Errorf("invalid metadata_max_size: %w", err)
}
if size <= 0 {
return fmt.Errorf("invalid metadata_max_size %q: must be positive", s)
}
return nil
}
func (c *Config) validateDownloadSizeLimits() error {
if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil {
return err
}
return validateArtifactMaxSize(c.ArtifactMaxSize)
}
func validateArtifactMaxSize(s string) error {
if s == "" {
return nil
}
size, err := ParseSize(s)
if err != nil {
return fmt.Errorf("invalid artifact_max_size: %w", err)
}
if size < 0 {
return fmt.Errorf("invalid artifact_max_size %q: must not be negative", s)
}
return nil
}
Cache rewritten npm and Composer metadata (#402) The npm and Composer handlers rewrite every metadata document they serve so that download URLs point at the proxy: decode the whole document into generic maps, change the URLs, encode it again, and for Composer expand the minified format first. That ran on every request, cached metadata included. With metadata caching on and upstream out of the picture, a cached request still cost 7 ms for @babel/core, 17 ms for symfony/console and 155 ms and 134 MB of allocations for typescript, and an 8 vCPU VM serving cached npm packuments to 20 clients ran the proxy at 450% CPU for about 1,100 requests a second. Rewritten documents are now kept in memory, keyed by the ecosystem, proxy URL, package and a SHA-256 of the raw document, so new bytes from upstream are rewritten again and nothing is served stale. Requests that arrive while a document is being rewritten wait for that rewrite rather than running their own; a waiter leaves when its client does, and the rewrite still completes and is cached. The denylist is fixed at startup, so it needs no place in the key. Cooldown filtering depends on the current time, so with cooldown on the cache is bypassed. metadata_rewrite_cache_size bounds the cache (default "256MB", least recently used out first, "0" to rewrite on every request). NewProxy callers keep the old behaviour unless they set it. Whole cached requests, measured locally: @babel/core 7.1 ms -> 0.38 ms, 54,489 -> 119 allocations typescript 155 ms -> 10.5 ms, 1.37M -> 133 allocations symfony/console 17.4 ms -> 0.48 ms, 165,909 -> 123 allocations What remains is reading the raw document from storage and hashing it.
2026-10-03 04:09:15 -07:00
func validateMetadataRewriteCacheSize(s string) error {
if s == "" || s == "0" {
return nil
}
size, err := ParseSize(s)
if err != nil {
return fmt.Errorf("invalid metadata_rewrite_cache_size: %w", err)
}
if size < 0 {
return fmt.Errorf("invalid metadata_rewrite_cache_size %q: must not be negative", s)
}
return nil
}
// ParseMetadataRewriteCacheSize returns how many bytes of rewritten metadata
// to keep in memory. Returns 256MB if unset or invalid, 0 if disabled.
func (c *Config) ParseMetadataRewriteCacheSize() int64 {
if c.MetadataRewriteCacheSize == "" {
return defaultMetadataRewriteCacheSize
}
if c.MetadataRewriteCacheSize == "0" {
return 0
}
size, err := ParseSize(c.MetadataRewriteCacheSize)
if err != nil || size < 0 {
return defaultMetadataRewriteCacheSize
}
return size
}
// ParseMetadataMaxSize returns the maximum metadata response size in bytes.
// Returns 100MB if unset or invalid.
func (c *Config) ParseMetadataMaxSize() int64 {
if c.MetadataMaxSize == "" {
return defaultMetadataMaxSize
}
size, err := ParseSize(c.MetadataMaxSize)
if err != nil || size <= 0 {
return defaultMetadataMaxSize
}
return size
}
// ParseArtifactMaxSize returns the artifact size limit in bytes, or 0 for
// no limit.
func (c *Config) ParseArtifactMaxSize() int64 {
if c.ArtifactMaxSize == "" {
return 0
}
size, err := ParseSize(c.ArtifactMaxSize)
if err != nil || size < 0 {
return 0
}
return size
}
// ParseHTTPTimeout returns the upstream HTTP client timeout.
// Returns 30s if unset, 0 (no timeout) if explicitly set to "0".
func (c *Config) ParseHTTPTimeout() time.Duration {
if c.HTTPTimeout == "" {
return defaultHTTPTimeout
}
if c.HTTPTimeout == "0" {
return 0
}
d, err := time.ParseDuration(c.HTTPTimeout)
if err != nil || d < 0 {
return defaultHTTPTimeout
}
return d
}
func validateHitFlushInterval(s string) error {
if s == "" || s == "0" {
return nil
}
d, err := time.ParseDuration(s)
if err != nil {
return fmt.Errorf("invalid database.hit_flush_interval %q: %w", s, err)
}
if d < 0 {
return fmt.Errorf("invalid database.hit_flush_interval %q: must be non-negative", s)
}
return nil
}
// ParseHitFlushInterval returns how often batched cache hits are written.
// Returns 1 second if unset or invalid, 0 if explicitly disabled.
func (c *Config) ParseHitFlushInterval() time.Duration {
if c.Database.HitFlushInterval == "" {
return defaultHitFlushInterval
}
if c.Database.HitFlushInterval == "0" {
return 0
}
d, err := time.ParseDuration(c.Database.HitFlushInterval)
if err != nil || d < 0 {
return defaultHitFlushInterval
}
return d
}
// ParseMetadataTTL returns the metadata TTL duration.
// Returns 5 minutes if unset, 0 if explicitly disabled.
func (c *Config) ParseMetadataTTL() time.Duration {
if c.MetadataTTL == "" {
return defaultMetadataTTL
}
if c.MetadataTTL == "0" {
return 0
}
d, err := time.ParseDuration(c.MetadataTTL)
if err != nil {
return defaultMetadataTTL
}
return d
}
// ParseGradleBuildCacheMaxUploadSize returns the max accepted PUT body size.
// Defaults to 100MB if unset or invalid.
func (c *Config) ParseGradleBuildCacheMaxUploadSize() int64 {
if c.Gradle.BuildCache.MaxUploadSize == "" {
return defaultGradleBuildCacheMaxUploadSize
}
size, err := ParseSize(c.Gradle.BuildCache.MaxUploadSize)
if err != nil || size <= 0 {
return defaultGradleBuildCacheMaxUploadSize
}
return size
}
// ParseGradleBuildCacheMaxAge returns age-based eviction threshold.
// Returns 0 when disabled or invalid.
func (c *Config) ParseGradleBuildCacheMaxAge() time.Duration {
if c.Gradle.BuildCache.MaxAge == "" || c.Gradle.BuildCache.MaxAge == "0" {
return 0
}
Add age-based retention for cached artifacts (#428) * Add age-based retention for cached artifacts Cached artifacts stayed forever unless storage.max_size pushed them out. storage.retention adds a default, per-ecosystem and per-package duration after which an artifact nobody downloaded gets evicted. Ecosystems opt in one at a time: a handler registers with retention.Default, and until it does, configuring that ecosystem (or one of its packages) fails validation, and the default doesn't touch it. This commit registers none, so nothing changes yet. Startup logs which ecosystems the default covers and which it doesn't. An artifact counts as expired when both its fetch time and its last access are older than the cutoff. Checking only the last access would evict a refetched artifact again right away, since clearing a record keeps its old access time and a cache miss doesn't record a hit. The sweep scans artifacts in id windows, so no single query holds the SQLite connection for long, and checks each row against its own rule. Expired records are cleared and their files go through pending_deletes instead of being deleted directly, because a buffered hit can make an artifact that's being served right now look expired. Buffered hits are flushed before each sweep, and one sweep clears at most 10,000 artifacts. Since a big first sweep can queue a lot, reclaim now keeps working through due entries for up to 30 seconds per tick instead of stopping after 100, and pending_deletes gets an index on queued_at. Also: - proxy_artifacts_evicted_total{reason, ecosystem} counts LRU and retention evictions. - gradle.build_cache.max_age accepts "7d", as its comment always said. Refs #306 * Document storage retention Describes the new storage.retention block, the order rules apply in, that the default only covers ecosystems that support retention yet, and what to expect from the first sweep over an old cache. Also lists the new eviction counter and notes that gradle max_age takes days. Refs #306 * Fix retention timing on Postgres and tighten the sweep On Postgres, fetched_at and last_accessed_at are TIMESTAMP columns without a zone. They hold the local time the proxy wrote, but lib/pq reads them back labelled UTC. East of UTC that makes every artifact look hours younger than it is, so the sweep evicted late: two hours in Berlin, nine in Tokyo. The candidates' times are now read back as local time, and the database tests run on both SQLite and Postgres, with one that sets the local zone to Asia/Tokyo. A few smaller things: - When a delete fails, reclaim stops for the rest of the minute instead of trying every queued path. Otherwise a storage outage turns into thirty seconds of failed deletes and warnings every minute. - If the buffered hits can't be written, the sweep skips that round. Running on stale access times could clear something that is being downloaded right now. - DefaultCanonical only accepts a package key when rebuilding it gives the same PURL. For alpine and deb it would otherwise produce keys that never match. - sweep_interval has to be at least a minute. - New tests cover the eviction counter and an artifact that gets a hit between the scan and the clear. - The docs no longer say nothing changes without retention, and they point out that the example's ecosystem and package rules only pass validation once those ecosystems support retention. Refs #306 * Reject qualifiers and subpaths in retention package keys A rule names a whole package. Qualifiers like ?type=pom were dropped on the way to the stored PURL and the rule quietly covered every file of the package. Now such a key fails validation. Refs #306 * Say when a package key doesn't match how packages are stored A package rule for an ecosystem that supports retention, written in a form its packages aren't stored under (pkg:npm/babel/core without the @), failed with "retention is not supported for npm packages". Once the ecosystem is enabled that is no longer true. Such a key now fails with "does not match how npm packages are stored" and points to the package rule forms in the docs. Keys for ecosystems without retention support keep the old message. Refs #306 * Note that re-running the mirror keeps packages cached A mirror run over a package that is still cached goes through the normal cache lookup and records a download, so the package's age starts over. The docs only said that mirrored packages age from the time they were mirrored. Refs #306 * Say when an ecosystem takes no package rules Some ecosystems will support retention without package rules, because nothing in their stored records names a package the way a PURL would (conan, helm). A package key for one of them used to fail with "does not match how conan packages are stored; see the package rule forms", which sends people looking for a form that doesn't exist. It now says package rules aren't supported there and points to the ecosystem rule. Refs #306 * Use helm as the example of an ecosystem without package rules Conan gets package rules once its handler stores archives under the recipe name, so the config test that shows the refusal now uses helm, whose cache records name the repository, not the chart. Refs #306 * Say that each cached file ages on its own The retention docs said each artifact, each version of a package, ages on its own. A version can have several cached files (a jar and its pom, several wheels, Conan sources and binaries), and each of them ages separately. Refs #306 * Clear an expired artifact and queue its file in one transaction The sweep cleared the record and queued the file for deletion as two separate writes. If queueing failed, or the process stopped in between, the file stayed in storage with no record pointing at it and no queue entry, and no later sweep would find it again. Both writes now share a transaction, so a failed queue rolls the clear back and the next sweep retries. Refs #306 * Show artifact evictions on the analytics page proxy_artifacts_evicted_total had no place on /ui/analytics. The Runtime card now lists evicted artifacts by reason and ecosystem, without the failure tint, and the coverage test records an eviction so it doesn't depend on test order. Refs #306 * Store artifact times in UTC fetched_at and last_accessed_at are now written in UTC, and the retention cutoffs are compared in UTC, the same convention #434 uses for the metadata cache. lib/pq sends a time with its offset and a Postgres TIMESTAMP keeps only the wall clock, so UTC on write is what reads back correctly. That replaces reinterpreting the times as local on read. Rows written before this still hold local time. East of UTC they look younger by the offset until they are rewritten, so retention evicts them a few hours late once; west of UTC that much early. Refs #306 --------- Co-authored-by: t <t@t>
2026-10-12 10:06:01 +02:00
d, err := parseDayDuration(c.Gradle.BuildCache.MaxAge)
if err != nil || d <= 0 {
return 0
}
return d
}
// ParseGradleBuildCacheMaxSize returns total-size cap in bytes.
// Returns 0 when disabled or invalid.
func (c *Config) ParseGradleBuildCacheMaxSize() int64 {
if c.Gradle.BuildCache.MaxSize == "" || c.Gradle.BuildCache.MaxSize == "0" {
return 0
}
size, err := ParseSize(c.Gradle.BuildCache.MaxSize)
if err != nil || size <= 0 {
return 0
}
return size
}
// ParseGradleBuildCacheSweepInterval returns eviction sweep cadence.
// Defaults to 10m if unset or invalid.
func (c *Config) ParseGradleBuildCacheSweepInterval() time.Duration {
if c.Gradle.BuildCache.SweepInterval == "" {
return defaultGradleBuildCacheSweepInterval
}
d, err := time.ParseDuration(c.Gradle.BuildCache.SweepInterval)
if err != nil || d <= 0 {
return defaultGradleBuildCacheSweepInterval
}
return d
}
// ParseDirectServeTTL returns the presigned URL expiry duration.
// Returns 15 minutes if unset.
func (c *Config) ParseDirectServeTTL() time.Duration {
if c.Storage.DirectServeTTL == "" {
return defaultDirectServeTTL
}
d, err := time.ParseDuration(c.Storage.DirectServeTTL)
if err != nil {
return defaultDirectServeTTL
}
return d
}
2026-01-20 21:52:44 +00:00
// ParseSize parses a human-readable size string (e.g., "10GB", "500MB").
// Returns the size in bytes.
func ParseSize(s string) (int64, error) {
s = strings.TrimSpace(strings.ToUpper(s))
if s == "" || s == "0" {
return 0, nil
}
// Check suffixes in order of length (longest first) to avoid partial matches
suffixes := []struct {
suffix string
mult int64
}{
{"TB", 1024 * 1024 * 1024 * 1024},
{"GB", 1024 * 1024 * 1024},
{"MB", 1024 * 1024},
{"KB", 1024},
{"T", 1024 * 1024 * 1024 * 1024},
{"G", 1024 * 1024 * 1024},
{"M", 1024 * 1024},
{"K", 1024},
{"B", 1},
}
for _, s2 := range suffixes {
if numStr, ok := strings.CutSuffix(s, s2.suffix); ok {
2026-01-20 21:52:44 +00:00
num, err := strconv.ParseFloat(numStr, 64)
if err != nil {
return 0, fmt.Errorf("invalid number %q", numStr)
}
return int64(num * float64(s2.mult)), nil
}
}
// Try parsing as plain number (bytes)
num, err := strconv.ParseInt(s, 10, 64)
if err != nil {
return 0, fmt.Errorf("invalid size %q", s)
}
return num, nil
}
// Header returns the HTTP header name and value for this auth config.
// Returns empty strings if the config is invalid or incomplete.
func (a *AuthConfig) Header() (name, value string) {
switch strings.ToLower(a.Type) {
case "bearer":
token := expandEnv(a.Token)
if token == "" {
return "", ""
}
return "Authorization", "Bearer " + token
case "basic":
username := expandEnv(a.Username)
password := expandEnv(a.Password)
if username == "" {
return "", ""
}
encoded := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
return "Authorization", "Basic " + encoded
case "header":
name := a.HeaderName
value := expandEnv(a.HeaderValue)
if name == "" {
return "", ""
}
return name, value
default:
return "", ""
}
}
// expandEnv expands ${VAR_NAME} references in a string.
func expandEnv(s string) string {
return os.Expand(s, os.Getenv)
}
func envBool(v string) bool {
return v == "true" || v == "1"
}