Watch
1
0
Fork
You've already forked pkg-proxy
1
mirror of https://github.com/git-pkgs/proxy.git synced 2026-11-09 23:59:16 -05:00
pkg-proxy/internal/config/config.go
simon chrzanowski 69313c26d9
Serve hosted-only npm scopes and Composer vendors from pushed packages alone
A package the upstream doesn't know is served from its pushed versions, but the upstream is still asked about it. Once someone publishes the same name there, their versions are listed next to the pushed ones and a version range can resolve to them.

hosted.npm_scopes and hosted.composer_vendors (PROXY_HOSTED_NPM_SCOPES, PROXY_HOSTED_COMPOSER_VENDORS) name namespaces whose packages are served from pushed versions alone. Their metadata is built from what was pushed, npm latest included, and anything not pushed answers 404. The upstream is never asked about them. Packages in other namespaces are proxied as before.

The npm metadata handler's decisions about pushed versions move into serveLocalMetadata.

Refs #133
Refs #134
2026-10-14 13:23:19 +02:00

1784 lines
62 KiB
Go

// Package config provides configuration loading and validation for the proxy server.
//
// Configuration can be provided via:
// - Command line flags (highest priority)
// - Environment variables (PROXY_ prefix)
// - Configuration file (YAML or JSON)
//
// Storage Configuration:
//
// The proxy supports multiple storage backends via gocloud.dev/blob:
//
// Local filesystem (default):
//
// storage:
// url: "file:///var/cache/proxy"
//
// Amazon S3:
//
// storage:
// url: "s3://bucket-name"
//
// S3-compatible (MinIO, etc.):
//
// storage:
// url: "s3://bucket?endpoint=http://localhost:9000"
//
// Google Cloud Storage:
//
// storage:
// url: "gs://bucket-name"
//
// For S3, configure credentials via AWS environment variables:
//
// AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
//
// For GCS, authentication uses Application Default Credentials. This supports
// GKE Workload Identity, attached service accounts on GCE and Cloud Run, and
// local credentials created by `gcloud auth application-default login`.
// When direct_serve is enabled without a private key, the GCS backend uses the
// IAM Credentials signBlob API. The service account must hold
// roles/iam.serviceAccountTokenCreator on itself.
//
// Database Configuration:
//
// The proxy supports two database backends:
//
// SQLite (default):
//
// database:
// driver: "sqlite"
// path: "/var/lib/proxy/cache.db"
//
// PostgreSQL:
//
// database:
// driver: "postgres"
// url: "postgres://user:password@localhost:5432/proxy?sslmode=disable"
//
// See config.example.yaml in the repository root for a complete example.
package config
import (
"encoding/base64"
"encoding/json"
"fmt"
"net"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
"github.com/git-pkgs/proxy/internal/denylist"
"github.com/git-pkgs/proxy/internal/retention"
"github.com/git-pkgs/purl"
"gopkg.in/yaml.v3"
)
// DefaultSwiftUpstream is the Swift Package Registry used when none is configured.
const DefaultSwiftUpstream = "https://tuist.dev/api/registry/swift"
// Config holds all configuration for the proxy server.
type Config struct {
// Listen is the address to listen on (e.g., ":8080", "127.0.0.1:8080").
Listen string `json:"listen" yaml:"listen"`
// BaseURL is the public URL where package endpoints are reachable.
// Used for rewriting package metadata URLs and shown to humans on the
// install guide so they know what to point their package manager at.
// Example: "https://proxy.example.com" or "http://localhost:8080"
BaseURL string `json:"base_url" yaml:"base_url"`
// UIBaseURL is the public URL where the web UI is reachable. Defaults to
// BaseURL when unset. Set this separately when the UI is served on a
// different hostname than the package endpoints — for example, the UI on a
// public domain behind auth while build machines hit a Docker network alias
// for the package endpoints.
// Example: "https://proxy.example.com/ui"
UIBaseURL string `json:"ui_base_url" yaml:"ui_base_url"`
// TrustForwardedFor attributes requests to the leftmost X-Forwarded-For
// entry instead of the TCP peer address, in the structured log, the access
// log and the request-source table on /ui/analytics.
//
// Enable this only when the proxy sits behind a load balancer or ingress
// that sets the header, because any client can send it: behind one it is
// the only way to see past the hop, in front of one it lets a caller choose
// what address it is attributed to.
TrustForwardedFor bool `json:"trust_forwarded_for" yaml:"trust_forwarded_for"`
// UIRequestSources shows the request-source table on /ui/analytics, which
// reports caller addresses, the tool each ran and how much each pulled.
//
// Off by default. The proxy has no authentication of its own, so until this
// is enabled /ui exposes what is cached rather than who called.
UIRequestSources bool `json:"ui_request_sources" yaml:"ui_request_sources"`
// UIFailureDetails lists the latest failures behind each failure counter on
// /ui/analytics: the package, its upstream URL or storage path, and the
// error. URLs are shown without credentials or query strings.
//
// Off by default, for the same reason as UIRequestSources: package names and
// upstream hosts say what the proxy's users depend on.
UIFailureDetails bool `json:"ui_failure_details" yaml:"ui_failure_details"`
// Storage configures artifact storage.
Storage StorageConfig `json:"storage" yaml:"storage"`
// Database configures the cache database.
Database DatabaseConfig `json:"database" yaml:"database"`
// Log configures logging.
Log LogConfig `json:"log" yaml:"log"`
// AccessLog configures the JSONL activity log.
AccessLog AccessLogConfig `json:"access_log" yaml:"access_log"`
// Upstream configures upstream registry URLs (optional overrides).
Upstream UpstreamConfig `json:"upstream" yaml:"upstream"`
// Cooldown configures version age filtering to mitigate supply chain attacks.
Cooldown CooldownConfig `json:"cooldown" yaml:"cooldown"`
// Denylist blocks explicitly configured package versions.
Denylist DenylistConfig `json:"denylist" yaml:"denylist"`
// Scanning configures pre-cache artifact scanning (trivy, ClamAV, Wiz,
// or a custom service) to mitigate supply chain attacks.
Scanning ScanningConfig `json:"scanning" yaml:"scanning"`
// CacheMetadata enables caching of upstream metadata responses for offline fallback.
// When enabled, metadata is stored in the database and storage backend.
// The mirror command always enables this regardless of this setting.
CacheMetadata bool `json:"cache_metadata" yaml:"cache_metadata"`
// MetadataTTL is how long cached metadata is considered fresh before
// revalidating with upstream. Uses Go duration syntax (e.g. "5m", "1h").
// Default: "5m". Set to "0" to always revalidate.
MetadataTTL string `json:"metadata_ttl" yaml:"metadata_ttl"`
// MetadataMaxSize is the maximum size of an upstream metadata response
// the proxy will buffer (e.g. "100MB", "250MB"). Responses over this
// size return ErrMetadataTooLarge. Default: "100MB".
MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"`
// ArtifactMaxSize is the largest upstream artifact the proxy will fetch
// (e.g. "2GB"). Larger artifacts are refused and not cached. Default: ""
// (unlimited). "0" is also unlimited.
ArtifactMaxSize string `json:"artifact_max_size" yaml:"artifact_max_size"`
// MetadataRewriteCacheSize is how much rewritten npm and Composer
// metadata to keep in memory, so a document is rewritten once rather than
// on every request (e.g. "256MB", "1GB"). Default: "256MB". Set to "0" to
// rewrite on every request.
MetadataRewriteCacheSize string `json:"metadata_rewrite_cache_size" yaml:"metadata_rewrite_cache_size"`
// HTTPTimeout is the timeout for individual upstream HTTP requests made
// by protocol handlers (metadata fetches, pass-through file requests).
// Uses Go duration syntax (e.g. "30s", "2m"). Default: "30s".
// Set to "0" to disable the timeout entirely. Independently of this
// setting, the shared transport gives up on an upstream that has not sent
// response headers within 60 seconds.
HTTPTimeout string `json:"http_timeout" yaml:"http_timeout"`
// MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP.
// Disabled by default to prevent unauthenticated users from triggering downloads.
MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"`
// Hosted configures pushing packages to the proxy (/api/hosted).
Hosted HostedConfig `json:"hosted" yaml:"hosted"`
// Gradle configures Gradle HttpBuildCache behavior.
Gradle GradleConfig `json:"gradle" yaml:"gradle"`
// Health configures the /health endpoint behavior.
Health HealthConfig `json:"health" yaml:"health"`
}
// CooldownConfig configures version cooldown periods.
// Versions published more recently than the cooldown are hidden from metadata responses.
type CooldownConfig struct {
// Default is the global default cooldown (e.g., "3d", "48h", "0" to disable).
Default string `json:"default" yaml:"default"`
// Ecosystems overrides the default for specific ecosystems.
Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"`
// Packages overrides the cooldown for specific packages (keyed by PURL).
// Valid PURL keys are normalized to canonical form before use.
Packages map[string]string `json:"packages" yaml:"packages"`
// PackagePatterns overrides the cooldown for packages whose PURLs match a glob.
// Exact package overrides take precedence over matching patterns.
PackagePatterns map[string]string `json:"package_patterns" yaml:"package_patterns"`
}
// NormalizedPackages returns a copy of the package overrides with valid PURL
// keys in canonical form. An explicitly canonical key wins over an equivalent
// noncanonical key, and invalid keys are preserved unchanged.
func (c *CooldownConfig) NormalizedPackages() map[string]string {
if c == nil || c.Packages == nil {
return nil
}
keys := make([]string, 0, len(c.Packages))
for key := range c.Packages {
keys = append(keys, key)
}
sort.Strings(keys)
normalized := make(map[string]string, len(c.Packages))
for _, key := range keys {
canonical := key
if parsed, err := purl.Parse(key); err == nil {
canonical = parsed.String()
}
if _, exists := normalized[canonical]; exists && key != canonical {
continue
}
normalized[canonical] = c.Packages[key]
}
return normalized
}
// ScanningConfig configures pre-cache artifact scanning (e.g. trivy,
// ClamAV, Wiz, or a custom service) to mitigate supply chain attacks.
// Unlike Cooldown, which only looks at a version's publish timestamp,
// scanning inspects the actual artifact bytes before they become
// servable from cache.
type ScanningConfig struct {
// Enabled turns on the scan gate. When false (default), artifacts are
// cached exactly as if scanning didn't exist.
Enabled bool `json:"enabled" yaml:"enabled"`
// FailOpen treats scanner errors and timeouts as an allow verdict
// instead of a block. Default is fail-closed, since the default
// posture for a security gate should block on infrastructure failure.
FailOpen bool `json:"fail_open" yaml:"fail_open"`
// Timeout bounds each scan call. Uses Go duration syntax (e.g. "30s").
// Default: "30s".
Timeout string `json:"timeout" yaml:"timeout"`
// SigningKey authenticates pull requests to the internal scan-fetch
// route used by every storage backend. Required whenever Enabled is
// true. Supports ${VAR_NAME} expansion like AuthConfig fields.
SigningKey string `json:"signing_key" yaml:"signing_key"`
// FetchBaseURL is the address scanners use to reach this proxy to pull
// staged artifacts. Defaults to BaseURL. Set this separately when
// scanners reach the proxy over an internal address different from the
// public-facing BaseURL (mirrors DirectServeBaseURL/UIBaseURL).
FetchBaseURL string `json:"fetch_base_url" yaml:"fetch_base_url"`
// Scanners is the list of external scanning services to call.
Scanners []ScannerConfig `json:"scanners" yaml:"scanners"`
}
// ScannerConfig configures a single external scanning service.
type ScannerConfig struct {
// Name identifies this scanner in logs and metrics.
Name string `json:"name" yaml:"name"`
// URL is the endpoint the proxy POSTs scan notifications to.
URL string `json:"url" yaml:"url"`
// Mode is "block" (default) or "monitor". A "block" scanner's verdict
// can prevent caching; a "monitor" scanner's findings are logged but
// never gate caching.
Mode string `json:"mode" yaml:"mode"`
// Ecosystems restricts this scanner to specific ecosystems (e.g.
// "npm", "pypi"). Empty means all ecosystems.
Ecosystems []string `json:"ecosystems" yaml:"ecosystems"`
// Headers are additional HTTP headers sent with every scan request
// (e.g. for authenticating to the scanner service). Values support
// ${VAR_NAME} expansion like AuthConfig fields.
Headers map[string]string `json:"headers" yaml:"headers"`
}
// SigningKeyExpanded returns SigningKey with ${VAR_NAME} references expanded.
func (s *ScanningConfig) SigningKeyExpanded() string {
return expandEnv(s.SigningKey)
}
// HeadersExpanded returns Headers with ${VAR_NAME} references expanded in
// each value.
func (s *ScannerConfig) HeadersExpanded() map[string]string {
if len(s.Headers) == 0 {
return nil
}
expanded := make(map[string]string, len(s.Headers))
for k, v := range s.Headers {
expanded[k] = expandEnv(v)
}
return expanded
}
// Validate checks the scanning configuration for errors, applying the
// default timeout if unset.
func (s *ScanningConfig) Validate() error {
if !s.Enabled {
return nil
}
if s.SigningKeyExpanded() == "" {
return fmt.Errorf("scanning.signing_key is required when scanning.enabled is true")
}
if len(s.Scanners) == 0 {
return fmt.Errorf("scanning.scanners must not be empty when scanning.enabled is true")
}
if s.FetchBaseURL != "" {
if err := validateAbsoluteURL("scanning.fetch_base_url", s.FetchBaseURL); err != nil {
return err
}
}
if s.Timeout == "" {
s.Timeout = defaultScanningTimeoutStr
}
if d, err := time.ParseDuration(s.Timeout); err != nil {
return fmt.Errorf("invalid scanning.timeout %q: %w", s.Timeout, err)
} else if d <= 0 {
return fmt.Errorf("invalid scanning.timeout %q: must be > 0", s.Timeout)
}
for i := range s.Scanners {
if err := s.Scanners[i].Validate(); err != nil {
return fmt.Errorf("scanning.scanners[%d]: %w", i, err)
}
}
return nil
}
// Validate checks a single scanner's configuration, applying the default
// mode ("block") if unset.
func (s *ScannerConfig) Validate() error {
if s.Name == "" {
return fmt.Errorf("name is required")
}
if err := validateAbsoluteURL("url", s.URL); err != nil {
return err
}
if s.Mode == "" {
s.Mode = "block"
}
switch s.Mode {
case "block", "monitor":
default:
return fmt.Errorf("invalid mode %q (must be block or monitor)", s.Mode)
}
return nil
}
// StorageConfig configures artifact storage.
type StorageConfig struct {
// URL is the storage backend URL.
// Supported schemes:
// - file:///path/to/dir - Local filesystem (default)
// - s3://bucket-name - Amazon S3
// - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
// - gs://bucket-name - Google Cloud Storage (Workload Identity supported)
// - azblob://container-name - Azure Blob Storage
// If empty, defaults to file:// with the Path value.
URL string `json:"url" yaml:"url"`
// Path is the directory where cached artifacts are stored.
// If URL is empty, this is used as file://{Path}.
//
// Deprecated: Use URL with file:// scheme instead.
Path string `json:"path" yaml:"path"`
// MaxSize is the maximum cache size (e.g., "10GB", "500MB").
// When exceeded, least recently used artifacts are evicted.
// Empty or "0" means unlimited.
MaxSize string `json:"max_size" yaml:"max_size"`
// DirectServe enables redirecting cached artifact downloads to presigned
// storage URLs (HTTP 302) instead of streaming bytes through the proxy.
// Only effective for backends that support URL signing (S3, GCS, Azure).
DirectServe bool `json:"direct_serve" yaml:"direct_serve"`
// DirectServeTTL is how long presigned URLs remain valid.
// Uses Go duration syntax (e.g. "5m", "1h"). Default: "15m".
DirectServeTTL string `json:"direct_serve_ttl" yaml:"direct_serve_ttl"`
// DirectServeBaseURL overrides the scheme and host of presigned URLs
// before returning them to clients. Useful when the proxy reaches
// storage at an internal address (e.g. 127.0.0.1 or a Docker hostname)
// but clients must use a public one.
DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"`
// CacheArtifacts stores fetched artifacts so later downloads are served
// from storage. When false, every download streams from upstream and
// nothing is stored; metadata is still cached, and cooldown and the
// denylist still apply. Useful when another caching layer sits in front
// of the proxy. False is incompatible with scanning, direct_serve and
// mirror_api, which all depend on stored artifacts. Default: true.
CacheArtifacts bool `json:"cache_artifacts" yaml:"cache_artifacts"`
// Retention evicts cached artifacts that have not been accessed for a
// configured time, per ecosystem and per package.
Retention RetentionConfig `json:"retention" yaml:"retention"`
}
// RetentionConfig configures age-based eviction of cached artifacts. An
// artifact's age is the time since it was last served, or since it was
// fetched when it has not been served since. Durations accept a "d" suffix
// for days; "0" or empty means never evict by age.
type RetentionConfig struct {
// Default applies to every ecosystem that supports retention and has no
// rule of its own.
Default string `json:"default" yaml:"default"`
// Ecosystems overrides the default per ecosystem key (e.g. "npm", "oci").
// Configuring an ecosystem whose retention is not supported yet is an
// error.
Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"`
// Packages overrides the ecosystem rule for single packages, keyed by
// package PURL.
Packages map[string]string `json:"packages" yaml:"packages"`
// SweepInterval is how often expired artifacts are looked for, at least
// one minute. Default: "10m".
SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"`
}
// GradleConfig configures Gradle-specific features.
type GradleConfig struct {
// BuildCache configures the /gradle HttpBuildCache endpoint.
BuildCache GradleBuildCacheConfig `json:"build_cache" yaml:"build_cache"`
}
// GradleBuildCacheConfig configures Gradle HttpBuildCache safeguards.
type GradleBuildCacheConfig struct {
// ReadOnly disables PUT uploads and keeps cache reads (GET/HEAD) enabled.
ReadOnly bool `json:"read_only" yaml:"read_only"`
// MaxUploadSize caps a single PUT body size (e.g., "100MB"). Must be > 0.
// Default: "100MB".
MaxUploadSize string `json:"max_upload_size" yaml:"max_upload_size"`
// MaxAge evicts entries older than this duration (e.g., "24h", "7d").
// Empty or "0" disables age-based eviction.
MaxAge string `json:"max_age" yaml:"max_age"`
// MaxSize evicts oldest entries until total Gradle cache size is <= MaxSize.
// Empty or "0" disables size-based eviction.
MaxSize string `json:"max_size" yaml:"max_size"`
// SweepInterval controls periodic eviction frequency.
// Default: "10m".
SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"`
}
// HealthConfig configures the /health endpoint.
type HealthConfig struct {
// StorageProbeInterval is the minimum time between storage backend probes.
// Uses Go duration syntax (e.g. "30s", "1m"). Default: "30s".
// Set to "0" to probe on every /health request (useful for low-traffic deployments).
StorageProbeInterval string `json:"storage_probe_interval" yaml:"storage_probe_interval"`
}
// DatabaseConfig configures the cache database.
type DatabaseConfig struct {
// Driver is the database driver: "sqlite" or "postgres".
Driver string `json:"driver" yaml:"driver"`
// Path is the path to the SQLite database file.
Path string `json:"path" yaml:"path"`
// URL is the PostgreSQL connection string.
URL string `json:"url" yaml:"url"`
// HitFlushInterval is how often cache hit counts and last-access times
// are written, batched in one transaction. Uses Go duration syntax
// (e.g. "1s", "10s"). Default: "1s". Set to "0" to write each hit as it
// happens.
HitFlushInterval string `json:"hit_flush_interval" yaml:"hit_flush_interval"`
}
// String returns a human-readable description of the configured database
// suitable for logging. For postgres the password in the connection URL is
// redacted; if the URL cannot be parsed only the driver name is returned to
// avoid leaking credentials.
func (d DatabaseConfig) String() string {
if d.Driver == "postgres" {
u, err := url.Parse(d.URL)
if err != nil || u.Host == "" {
return "postgres"
}
return u.Redacted()
}
return d.Path
}
// LogConfig configures logging.
type LogConfig struct {
// Level is the minimum log level: "debug", "info", "warn", "error".
Level string `json:"level" yaml:"level"`
// Format is the log format: "text" or "json".
Format string `json:"format" yaml:"format"`
}
// AccessLogConfig configures the JSONL activity log.
type AccessLogConfig struct {
// Path is the file to append activity records to. Empty disables the access log.
Path string `json:"path" yaml:"path"`
}
// UpstreamConfig configures upstream URLs for built-in routes and authentication.
// Leave empty to use defaults.
type UpstreamConfig struct {
// AllowPrivateHosts permits listed upstream hosts to resolve to private addresses.
AllowPrivateHosts []string `json:"allow_private_hosts" yaml:"allow_private_hosts"`
// AllowLoopback permits upstream requests and redirects to loopback addresses.
AllowLoopback bool `json:"allow_loopback" yaml:"allow_loopback"`
// NPM is the upstream npm registry URL.
// Default: https://registry.npmjs.org
NPM string `json:"npm" yaml:"npm"`
// NPMFullMetadata always requests the full packument (application/json)
// from the npm upstream, so served metadata carries the "time" map even
// when cooldown is disabled. Clients that gate on publish age (for
// example Yarn's npmMinimalAgeGate) need this.
// Default: false (the abbreviated format is preferred).
NPMFullMetadata bool `json:"npm_full_metadata" yaml:"npm_full_metadata"`
// NPMRoutes maps package name patterns to npm registry URLs. A package
// whose name matches a pattern is fetched only from that registry, never
// from upstream.npm, so a public package cannot stand in for it. Patterns
// use path.Match syntax, e.g. "@example/*"; the longest matching pattern
// wins.
NPMRoutes map[string]string `json:"npm_routes" yaml:"npm_routes"`
// Cargo is the upstream cargo index URL.
// Default: https://index.crates.io
Cargo string `json:"cargo" yaml:"cargo"`
// CargoDownload is the upstream cargo download URL.
// Default: https://static.crates.io/crates
CargoDownload string `json:"cargo_download" yaml:"cargo_download"`
// Gem is the upstream RubyGems registry URL.
// Default: https://rubygems.org
Gem string `json:"gem" yaml:"gem"`
// Go is the upstream Go module proxy URL.
// Default: https://proxy.golang.org
Go string `json:"go" yaml:"go"`
// GoSumDBs maps Go checksum database names to their base URLs, served at
// /go/sumdb/{name}/ so the go command reaches the database through
// GOPROXY. Unknown names get 404, which makes the go command contact the
// database directly.
// Default when empty: {"sum.golang.org": "https://sum.golang.org"}.
GoSumDBs map[string]string `json:"go_sumdbs" yaml:"go_sumdbs"`
// GoSumDBsOffline keeps answering for a checksum database from its
// cached signed tree, however old, when the database cannot be reached.
// Set this when clients have no route to the database of their own;
// otherwise the proxy stops answering once the cached tree is older than
// metadata_ttl, and the go command contacts the database directly.
// Requires cache_metadata. Default: false.
GoSumDBsOffline bool `json:"go_sumdbs_offline" yaml:"go_sumdbs_offline"`
// Hex is the upstream Hex repository URL.
// Default: https://repo.hex.pm
Hex string `json:"hex" yaml:"hex"`
// HexAPI is the upstream Hex API URL used for package timestamps.
// Default: https://hex.pm
HexAPI string `json:"hex_api" yaml:"hex_api"`
// Pub is the upstream pub registry URL.
// Default: https://pub.dev
Pub string `json:"pub" yaml:"pub"`
// PyPI is the upstream PyPI index and API URL.
// Default: https://pypi.org
PyPI string `json:"pypi" yaml:"pypi"`
// PyPIDownload is the upstream PyPI package download URL.
// Default: https://files.pythonhosted.org
PyPIDownload string `json:"pypi_download" yaml:"pypi_download"`
// Maven is the upstream Maven repository URL.
// Default: https://repo1.maven.org/maven2
Maven string `json:"maven" yaml:"maven"`
// GradlePluginPortal is the upstream Gradle Plugin Portal Maven URL.
// Used to resolve Gradle plugin marker artifacts.
// Default: https://plugins.gradle.org/m2
GradlePluginPortal string `json:"gradle_plugin_portal" yaml:"gradle_plugin_portal"`
// NuGet is the upstream NuGet API URL.
// Default: https://api.nuget.org
NuGet string `json:"nuget" yaml:"nuget"`
// NuGetSearch is the upstream NuGet search API URL.
// Default: https://azuresearch-usnc.nuget.org
NuGetSearch string `json:"nuget_search" yaml:"nuget_search"`
// Composer is the upstream Packagist API URL.
// Default: https://packagist.org
Composer string `json:"composer" yaml:"composer"`
// ComposerRepository is the upstream Packagist repository URL.
// Default: https://repo.packagist.org
ComposerRepository string `json:"composer_repository" yaml:"composer_repository"`
// ComposerRoutes maps package name patterns to Composer repository URLs
// (the base that serves /p2/{vendor}/{package}.json). A package whose
// name matches a pattern is fetched only from that repository, never
// from upstream.composer_repository, so a public package cannot stand in
// for it. Patterns use path.Match syntax, e.g. "example/*"; the longest
// matching pattern wins.
ComposerRoutes map[string]string `json:"composer_routes" yaml:"composer_routes"`
// Conan is the upstream Conan registry URL.
// Default: https://center.conan.io
Conan string `json:"conan" yaml:"conan"`
// Conda is the upstream Conda channel base URL.
// Default: https://conda.anaconda.org
Conda string `json:"conda" yaml:"conda"`
// CRAN is the upstream CRAN mirror URL.
// Default: https://cloud.r-project.org
CRAN string `json:"cran" yaml:"cran"`
// Julia is the upstream Julia package server URL.
// Default: https://pkg.julialang.org
Julia string `json:"julia" yaml:"julia"`
// OCIDefault is the default upstream OCI registry URL.
// Default: https://registry-1.docker.io
OCIDefault string `json:"oci_default" yaml:"oci_default"`
// Swift is the upstream Swift Package Registry URL.
// Default: https://tuist.dev/api/registry/swift
Swift string `json:"swift" yaml:"swift"`
// Debian is the upstream APT repository base URL, served at /debian/.
// Example: http://archive.ubuntu.com/ubuntu would get Ubuntu.
// Default: http://deb.debian.org/debian
Debian string `json:"debian" yaml:"debian"`
// DebianRepositories maps repository names to additional APT repository
// base URLs, served at /debian/{name}/.
// Example: {"security": "https://security.debian.org/debian-security"}.
DebianRepositories map[string]string `json:"debian_repositories" yaml:"debian_repositories"`
// RPM is the upstream RPM repository base URL.
// Default: https://dl.fedoraproject.org/pub/fedora/linux
RPM string `json:"rpm" yaml:"rpm"`
// HomebrewAPI is the upstream Homebrew JSON API URL.
// Default: https://formulae.brew.sh/api
HomebrewAPI string `json:"homebrew_api" yaml:"homebrew_api"`
// HomebrewArtifact is the upstream registry URL for Homebrew artifacts.
// Default: https://ghcr.io
HomebrewArtifact string `json:"homebrew_artifact" yaml:"homebrew_artifact"`
// Helm maps repository names to HTTP Helm chart repository URLs.
// Requests use /helm/{name}/index.yaml and chart URLs in the index are
// rewritten to the same named proxy endpoint.
Helm map[string]string `json:"helm" yaml:"helm"`
// APK maps repository names to Alpine APK repository base URLs, served
// at /apk/{name}/. The remaining request path mirrors the upstream
// layout, e.g. /apk/alpine/v3.22/main/x86_64/APKINDEX.tar.gz.
// Default when empty: {"alpine": "https://dl-cdn.alpinelinux.org/alpine"}.
APK map[string]string `json:"apk" yaml:"apk"`
// Arch maps repository names to Arch Linux pacman mirror base URLs,
// served at /arch/{name}/. The remaining request path mirrors the
// upstream layout, e.g. /arch/archlinux/core/os/x86_64/core.db.
// Default when empty: {"archlinux": "https://geo.mirror.pkgbuild.com"}.
Arch map[string]string `json:"arch" yaml:"arch"`
// OCI maps names to OCI registry URLs. Requests to a named registry use
// the repository prefix upstream/{name}/, for example
// oci://proxy.example.com/upstream/ghcr/owner/chart.
OCI map[string]string `json:"oci" yaml:"oci"`
// OCIMirrors lists, per upstream.oci name, the registry hosts that
// upstream mirrors, for example {"ghcr": ["ghcr.io"]} for an Artifactory
// remote of ghcr.io. containerd requests whose ns query parameter names
// one of these hosts are served by that upstream.
OCIMirrors map[string][]string `json:"oci_mirrors" yaml:"oci_mirrors"`
// Generic maps names to plain HTTP upstream base URLs, served at
// /generic/{name}/. The remaining request path and query string are
// appended to the upstream URL. GitHub release asset paths
// ({owner}/{repo}/releases/download/{tag}/{asset}) and paths matching
// GenericArtifacts are cached in the artifact cache; everything else
// goes through the metadata cache.
// Example: {"github": "https://github.com", "github-api": "https://api.github.com"}.
Generic map[string]string `json:"generic" yaml:"generic"`
// GenericArtifacts maps a Generic upstream name to regular expressions for
// its version-pinned download paths. A request path after
// /generic/{name}/ that one of them matches in full is cached in the
// artifact cache like a GitHub release asset. Each pattern needs `version`
// and `file` named groups; an optional `name` group names the package,
// which otherwise is the upstream name.
// Example: {"apache": ["dist/(?P<name>maven)/maven-3/(?P<version>[^/]+)/binaries/(?P<file>[^/]+)"]}.
GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"`
// Auth configures authentication for upstream registries.
// Keys are absolute URL scopes matched by scheme, host, effective port,
// and path-segment prefix.
// Example: "https://npm.pkg.github.com" matches all requests to that host.
Auth map[string]AuthConfig `json:"auth" yaml:"auth"`
}
// AuthForURL returns the auth config that matches the given URL.
// The longest matching URL scope wins.
func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig {
if u.Auth == nil {
return nil
}
target, err := parseAuthURL(url)
if err != nil {
return nil
}
var bestMatch *AuthConfig
var bestLen int
for pattern, auth := range u.Auth {
configured, err := parseAuthURL(pattern)
if err == nil && authURLMatches(configured, target) && len(pattern) > bestLen {
a := auth // copy to avoid loop variable capture
bestMatch = &a
bestLen = len(pattern)
}
}
return bestMatch
}
// Validate checks upstream authentication URL scopes.
func (u *UpstreamConfig) Validate() error {
for pattern := range u.Auth {
if _, err := parseAuthURL(pattern); err != nil {
return fmt.Errorf("invalid upstream.auth URL %q: %w", pattern, err)
}
}
if err := validateNamedUpstreams("upstream.helm", u.Helm); err != nil {
return err
}
if err := validatePackageRoutes("upstream.composer_routes", u.ComposerRoutes); err != nil {
return err
}
if err := validatePackageRoutes("upstream.npm_routes", u.NPMRoutes); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.apk", u.APK); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.go_sumdbs", u.GoSumDBs); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil {
return err
}
if err := validateOCIMirrors(u.OCIMirrors, u.OCI); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil {
return err
}
if err := u.validateGenericArtifacts(); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.debian_repositories", u.DebianRepositories); err != nil {
return err
}
for _, name := range debianReservedRepositoryNames {
if _, found := u.DebianRepositories[name]; found {
return fmt.Errorf(
"invalid upstream.debian_repositories name %q: reserved for the upstream.debian archive",
name,
)
}
}
return nil
}
func (u *UpstreamConfig) validateGenericArtifacts() error {
for name, patterns := range u.GenericArtifacts {
if _, found := u.Generic[name]; !found {
return fmt.Errorf("invalid upstream.generic_artifacts name %q: no upstream.generic entry", name)
}
for _, pattern := range patterns {
if _, err := CompileGenericArtifactPattern(pattern); err != nil {
return fmt.Errorf("invalid upstream.generic_artifacts.%s pattern %q: %w", name, pattern, err)
}
}
}
return nil
}
// validateUpstream checks the upstream block, including the one setting in
// it that depends on a top-level option.
func (c *Config) validateUpstream() error {
if err := c.Upstream.Validate(); err != nil {
return err
}
if c.Upstream.GoSumDBsOffline && !c.CacheMetadata {
return fmt.Errorf("upstream.go_sumdbs_offline requires cache_metadata")
}
return nil
}
// CompileGenericArtifactPattern compiles an upstream.generic_artifacts pattern
// so that it only matches a whole path, and checks it has the `version` and
// `file` groups.
func CompileGenericArtifactPattern(pattern string) (*regexp.Regexp, error) {
re, err := regexp.Compile(`^(?:` + pattern + `)$`)
if err != nil {
return nil, err
}
for _, group := range []string{"version", "file"} {
if re.SubexpIndex(group) < 0 {
return nil, fmt.Errorf("missing (?P<%s>...) group", group)
}
}
return re, nil
}
// debianReservedRepositoryNames are the upstream.debian archive's own root
// paths, which a repository of the same name would shadow.
var debianReservedRepositoryNames = []string{"pool", "dists"}
// validateOCIMirrors checks that every upstream.oci_mirrors entry belongs to
// an upstream.oci name and lists bare registry hosts, the form containerd
// sends in the ns query parameter. A host may belong to one upstream only;
// otherwise both prefixes would take it while unprefixed requests silently
// went to one of them.
func validateOCIMirrors(mirrors map[string][]string, upstreams map[string]string) error {
names := make([]string, 0, len(mirrors))
for name := range mirrors {
names = append(names, name)
}
sort.Strings(names)
owners := make(map[string]string)
for _, name := range names {
if _, ok := upstreams[name]; !ok {
return fmt.Errorf("invalid upstream.oci_mirrors name %q: no upstream.oci entry of that name", name)
}
for _, host := range mirrors[name] {
parsed, err := url.Parse("https://" + host)
if host == "" || err != nil || parsed.Host != host {
return fmt.Errorf("invalid upstream.oci_mirrors.%s host %q: must be a registry host such as ghcr.io or registry.example:5000", name, host)
}
key := ociMirrorHostKey(host)
if owner, taken := owners[key]; taken && owner != name {
return fmt.Errorf("invalid upstream.oci_mirrors.%s host %q: already listed for %s", name, host, owner)
}
owners[key] = name
}
}
return nil
}
// ociMirrorHostKey normalizes an upstream.oci_mirrors host the way the
// container handler builds its ns lookup keys: lowercase, IPv6 literals in
// brackets, an empty or default https port dropped. Two entries with the same
// key would select the same route.
func ociMirrorHostKey(host string) string {
name, port, err := net.SplitHostPort(host)
if err != nil {
name, port = strings.TrimSuffix(strings.TrimPrefix(host, "["), "]"), ""
}
name = strings.ToLower(name)
if strings.Contains(name, ":") {
name = "[" + name + "]"
}
if port == "" || port == "443" {
return name
}
return name + ":" + port
}
func validateNamedUpstreams(field string, upstreams map[string]string) error {
for name, upstreamURL := range upstreams {
if name == "" || name == "." || name == ".." || strings.ContainsAny(name, `/\\`) {
return fmt.Errorf("invalid %s name %q", field, name)
}
if err := validateAbsoluteURL(field+"."+name, upstreamURL); err != nil {
return err
}
}
return nil
}
func validatePackageRoutes(field string, routes map[string]string) error {
for pattern, upstreamURL := range routes {
if _, err := path.Match(pattern, ""); pattern == "" || err != nil {
return fmt.Errorf("invalid %s pattern %q", field, pattern)
}
if err := validateAbsoluteURL(field+"."+pattern, upstreamURL); err != nil {
return err
}
}
return nil
}
func parseAuthURL(value string) (*url.URL, error) {
parsed, err := url.Parse(value)
if err != nil || !parsed.IsAbs() || parsed.Hostname() == "" || parsed.Opaque != "" {
return nil, fmt.Errorf("invalid authentication URL")
}
return parsed, nil
}
func authURLMatches(configured, target *url.URL) bool {
if !strings.EqualFold(configured.Scheme, target.Scheme) ||
!strings.EqualFold(configured.Hostname(), target.Hostname()) ||
authURLPort(configured) != authURLPort(target) {
return false
}
if configured.RawQuery != "" && configured.RawQuery != target.RawQuery {
return false
}
configuredPath := strings.TrimSuffix(configured.EscapedPath(), "/")
if configuredPath == "" {
return true
}
targetPath := strings.TrimSuffix(target.EscapedPath(), "/")
return targetPath == configuredPath || strings.HasPrefix(targetPath, configuredPath+"/")
}
func authURLPort(value *url.URL) string {
if port := value.Port(); port != "" {
return port
}
if strings.EqualFold(value.Scheme, "https") {
return "443"
}
if strings.EqualFold(value.Scheme, "http") {
return "80"
}
return ""
}
// AuthConfig configures authentication for an upstream registry.
type AuthConfig struct {
// Type is the authentication type: "bearer", "basic", "header", or "ecr".
Type string `json:"type" yaml:"type"`
// Token is used for bearer authentication.
// Can reference environment variables with ${VAR_NAME} syntax.
Token string `json:"token" yaml:"token"`
// Username is used for basic authentication.
Username string `json:"username" yaml:"username"`
// Password is used for basic authentication.
// Can reference environment variables with ${VAR_NAME} syntax.
Password string `json:"password" yaml:"password"`
// HeaderName is the custom header name (for type "header").
HeaderName string `json:"header_name" yaml:"header_name"`
// HeaderValue is the custom header value (for type "header").
// Can reference environment variables with ${VAR_NAME} syntax.
HeaderValue string `json:"header_value" yaml:"header_value"`
// Region is the AWS region for ECR authentication (for type "ecr").
// If empty, the region is inferred from private ECR registry URLs before
// falling back to the AWS SDK default region chain.
Region string `json:"region" yaml:"region"`
}
// Default returns a Config with sensible defaults.
func Default() *Config {
return &Config{
Listen: ":8080",
BaseURL: "http://localhost:8080",
Storage: StorageConfig{
Path: "./cache/artifacts",
MaxSize: "",
CacheArtifacts: true,
Retention: RetentionConfig{
SweepInterval: defaultRetentionSweepIntervalStr,
},
},
Database: DatabaseConfig{
Driver: "sqlite",
Path: "./cache/proxy.db",
},
Log: LogConfig{
Level: "info",
Format: "text",
},
Upstream: UpstreamConfig{
NPM: "https://registry.npmjs.org",
Cargo: "https://index.crates.io",
CargoDownload: "https://static.crates.io/crates",
Gem: "https://rubygems.org",
Go: "https://proxy.golang.org",
Hex: "https://repo.hex.pm",
HexAPI: "https://hex.pm",
Pub: "https://pub.dev",
PyPI: "https://pypi.org",
PyPIDownload: "https://files.pythonhosted.org",
Maven: "https://repo1.maven.org/maven2",
GradlePluginPortal: "https://plugins.gradle.org/m2",
NuGet: "https://api.nuget.org",
NuGetSearch: "https://azuresearch-usnc.nuget.org",
Composer: "https://packagist.org",
ComposerRepository: "https://repo.packagist.org",
Conan: "https://center.conan.io",
Conda: "https://conda.anaconda.org",
CRAN: "https://cloud.r-project.org",
Julia: "https://pkg.julialang.org",
Swift: DefaultSwiftUpstream,
OCIDefault: "https://registry-1.docker.io",
Debian: "http://deb.debian.org/debian",
RPM: "https://dl.fedoraproject.org/pub/fedora/linux",
HomebrewAPI: "https://formulae.brew.sh/api",
HomebrewArtifact: "https://ghcr.io",
},
Gradle: GradleConfig{
BuildCache: GradleBuildCacheConfig{
ReadOnly: false,
MaxUploadSize: defaultGradleMaxUploadSizeStr,
MaxAge: "168h",
MaxSize: "",
SweepInterval: defaultGradleSweepIntervalStr,
},
},
}
}
// Load reads configuration from a file (YAML or JSON).
func Load(path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("reading config file: %w", err)
}
cfg := Default()
ext := strings.ToLower(filepath.Ext(path))
switch ext {
case ".yaml", ".yml":
if err := yaml.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("parsing YAML config: %w", err)
}
case ".json":
if err := json.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("parsing JSON config: %w", err)
}
default:
// Try YAML first, then JSON
if err := yaml.Unmarshal(data, cfg); err != nil {
if err := json.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("parsing config (tried YAML and JSON): %w", err)
}
}
}
return cfg, nil
}
// setEnvString sets *dst from the named environment variable, leaving it
// untouched if the variable is unset or empty.
func setEnvString(dst *string, key string) {
if v := os.Getenv(key); v != "" {
*dst = v
}
}
// setEnvBool is setEnvString for boolean fields, parsed via envBool.
func setEnvBool(dst *bool, key string) {
if v := os.Getenv(key); v != "" {
*dst = envBool(v)
}
}
func setEnvStringSlice(dst *[]string, key string) {
value := os.Getenv(key)
if value == "" {
return
}
var items []string
for item := range strings.SplitSeq(value, ",") {
if item = strings.TrimSpace(item); item != "" {
items = append(items, item)
}
}
if len(items) > 0 {
*dst = items
}
}
// LoadFromEnv applies environment variable overrides to a Config.
// Environment variables use the PROXY_ prefix:
// - PROXY_LISTEN
// - PROXY_BASE_URL
// - PROXY_UI_URL
// - PROXY_STORAGE_PATH
// - PROXY_STORAGE_MAX_SIZE
// - PROXY_DATABASE_PATH
// - PROXY_LOG_LEVEL
// - PROXY_LOG_FORMAT
// - PROXY_ACCESS_LOG_PATH
// - PROXY_TRUST_FORWARDED_FOR
// - PROXY_UI_REQUEST_SOURCES
// - PROXY_UPSTREAM_SWIFT
// - PROXY_HEALTH_STORAGE_PROBE_INTERVAL
func (c *Config) LoadFromEnv() {
setEnvString(&c.Listen, "PROXY_LISTEN")
setEnvString(&c.BaseURL, "PROXY_BASE_URL")
setEnvString(&c.UIBaseURL, "PROXY_UI_URL")
setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR")
setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES")
setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS")
setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL")
setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH")
setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE")
setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE")
setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL")
setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL")
setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS")
setEnvString(&c.Storage.Retention.Default, "PROXY_STORAGE_RETENTION_DEFAULT")
setEnvString(&c.Storage.Retention.SweepInterval, "PROXY_STORAGE_RETENTION_SWEEP_INTERVAL")
setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER")
setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH")
setEnvString(&c.Database.URL, "PROXY_DATABASE_URL")
setEnvString(&c.Database.HitFlushInterval, "PROXY_DATABASE_HIT_FLUSH_INTERVAL")
setEnvString(&c.Log.Level, "PROXY_LOG_LEVEL")
setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT")
setEnvString(&c.AccessLog.Path, "PROXY_ACCESS_LOG_PATH")
setEnvStringSlice(&c.Upstream.AllowPrivateHosts, "PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS")
setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK")
setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM")
setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA")
setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE")
setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO")
setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD")
setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM")
setEnvString(&c.Upstream.Go, "PROXY_UPSTREAM_GO")
setEnvString(&c.Upstream.Hex, "PROXY_UPSTREAM_HEX")
setEnvString(&c.Upstream.HexAPI, "PROXY_UPSTREAM_HEX_API")
setEnvString(&c.Upstream.Pub, "PROXY_UPSTREAM_PUB")
setEnvString(&c.Upstream.PyPI, "PROXY_UPSTREAM_PYPI")
setEnvString(&c.Upstream.PyPIDownload, "PROXY_UPSTREAM_PYPI_DOWNLOAD")
setEnvString(&c.Upstream.Maven, "PROXY_UPSTREAM_MAVEN")
setEnvString(&c.Upstream.GradlePluginPortal, "PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL")
setEnvString(&c.Upstream.NuGet, "PROXY_UPSTREAM_NUGET")
setEnvString(&c.Upstream.NuGetSearch, "PROXY_UPSTREAM_NUGET_SEARCH")
setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER")
setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY")
setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN")
setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA")
setEnvString(&c.Upstream.CRAN, "PROXY_UPSTREAM_CRAN")
setEnvString(&c.Upstream.Julia, "PROXY_UPSTREAM_JULIA")
setEnvString(&c.Upstream.Swift, "PROXY_UPSTREAM_SWIFT")
setEnvString(&c.Upstream.OCIDefault, "PROXY_UPSTREAM_OCI_DEFAULT")
setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN")
setEnvString(&c.Upstream.RPM, "PROXY_UPSTREAM_RPM")
setEnvString(&c.Upstream.HomebrewAPI, "PROXY_UPSTREAM_HOMEBREW_API")
setEnvString(&c.Upstream.HomebrewArtifact, "PROXY_UPSTREAM_HOMEBREW_ARTIFACT")
setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT")
setEnvBool(&c.Scanning.Enabled, "PROXY_SCANNING_ENABLED")
setEnvBool(&c.Scanning.FailOpen, "PROXY_SCANNING_FAIL_OPEN")
setEnvString(&c.Scanning.Timeout, "PROXY_SCANNING_TIMEOUT")
setEnvString(&c.Scanning.SigningKey, "PROXY_SCANNING_SIGNING_KEY")
setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL")
setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA")
setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API")
setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED")
setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN")
setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE")
setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES")
setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS")
setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL")
setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE")
setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE")
setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE")
setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT")
setEnvBool(&c.Gradle.BuildCache.ReadOnly, "PROXY_GRADLE_BUILD_CACHE_READ_ONLY")
setEnvString(&c.Gradle.BuildCache.MaxUploadSize, "PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE")
setEnvString(&c.Gradle.BuildCache.MaxAge, "PROXY_GRADLE_BUILD_CACHE_MAX_AGE")
setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE")
setEnvString(&c.Gradle.BuildCache.SweepInterval, "PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL")
setEnvString(&c.Health.StorageProbeInterval, "PROXY_HEALTH_STORAGE_PROBE_INTERVAL")
}
// validateAbsoluteURL returns an error if value is not a parseable URL with
// both a scheme and host. fieldName is used in the error message.
func validateAbsoluteURL(fieldName, value string) error {
u, err := url.Parse(value)
if err != nil || u.Scheme == "" || u.Host == "" {
return fmt.Errorf("invalid %s %q: must be an absolute URL", fieldName, value)
}
return nil
}
// Validate checks the configuration for errors.
func (c *Config) Validate() error {
if c.Listen == "" {
return fmt.Errorf("listen address is required")
}
if c.BaseURL == "" {
return fmt.Errorf("base_url is required")
}
if c.UIBaseURL == "" {
c.UIBaseURL = c.BaseURL
} else if err := validateAbsoluteURL("ui_base_url", c.UIBaseURL); err != nil {
return err
}
if c.Storage.URL == "" && c.Storage.Path == "" {
return fmt.Errorf("storage.url or storage.path is required")
}
switch c.Database.Driver {
case "sqlite":
if c.Database.Path == "" {
return fmt.Errorf("database.path is required for sqlite driver")
}
case "postgres":
if c.Database.URL == "" {
return fmt.Errorf("database.url is required for postgres driver")
}
default:
return fmt.Errorf("invalid database.driver %q (must be sqlite or postgres)", c.Database.Driver)
}
// Validate log level
switch strings.ToLower(c.Log.Level) {
case "debug", "info", "warn", "error":
// OK
default:
return fmt.Errorf("invalid log level %q (must be debug, info, warn, or error)", c.Log.Level)
}
// Validate log format
switch strings.ToLower(c.Log.Format) {
case "text", "json":
// OK
default:
return fmt.Errorf("invalid log format %q (must be text or json)", c.Log.Format)
}
// Validate max size if specified
if c.Storage.MaxSize != "" {
if _, err := ParseSize(c.Storage.MaxSize); err != nil {
return fmt.Errorf("invalid storage.max_size: %w", err)
}
}
// Validate direct serve TTL if specified
if c.Storage.DirectServeTTL != "" {
if _, err := time.ParseDuration(c.Storage.DirectServeTTL); err != nil {
return fmt.Errorf("invalid storage.direct_serve_ttl %q: %w", c.Storage.DirectServeTTL, err)
}
}
// Validate direct serve base URL if specified
if c.Storage.DirectServeBaseURL != "" {
if err := validateAbsoluteURL("storage.direct_serve_base_url", c.Storage.DirectServeBaseURL); err != nil {
return err
}
}
if err := c.validateCacheArtifacts(); err != nil {
return err
}
// Validate metadata TTL if specified
if c.MetadataTTL != "" && c.MetadataTTL != "0" {
if _, err := time.ParseDuration(c.MetadataTTL); err != nil {
return fmt.Errorf("invalid metadata_ttl %q: %w", c.MetadataTTL, err)
}
}
if err := c.validateDownloadSizeLimits(); err != nil {
return err
}
if err := validateHTTPTimeout(c.HTTPTimeout); err != nil {
return err
}
if err := validateHitFlushInterval(c.Database.HitFlushInterval); err != nil {
return err
}
return c.validateComponents()
}
func (c *Config) validateCacheArtifacts() error {
if c.Storage.CacheArtifacts {
return nil
}
switch {
case c.Scanning.Enabled:
return fmt.Errorf("storage.cache_artifacts: false cannot be combined with scanning.enabled: scanning needs stored artifacts")
case c.Storage.DirectServe:
return fmt.Errorf("storage.cache_artifacts: false cannot be combined with storage.direct_serve: no artifacts are stored to redirect to")
case c.MirrorAPI:
return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served")
case c.Hosted.Enabled:
return fmt.Errorf("storage.cache_artifacts: false cannot be combined with hosted.enabled: pushed packages would never be served")
}
return nil
}
func (c *Config) validateComponents() error {
if err := validateMetadataRewriteCacheSize(c.MetadataRewriteCacheSize); err != nil {
return err
}
if _, err := c.RetentionRules(retention.Default); err != nil {
return err
}
if _, err := denylist.New(c.Denylist.Packages); err != nil {
return err
}
if err := c.validateUpstream(); err != nil {
return err
}
if err := c.Health.Validate(); err != nil {
return err
}
if err := c.Scanning.Validate(); err != nil {
return err
}
if err := c.Hosted.Validate(); err != nil {
return err
}
return c.Gradle.BuildCache.Validate()
}
// Validate checks the /health configuration. An unset interval is allowed
// (the cache uses its default); explicit values must parse and be non-negative.
func (h *HealthConfig) Validate() error {
if h.StorageProbeInterval == "" || h.StorageProbeInterval == "0" {
return nil
}
d, err := time.ParseDuration(h.StorageProbeInterval)
if err != nil {
return fmt.Errorf("invalid health.storage_probe_interval %q: %w", h.StorageProbeInterval, err)
}
if d < 0 {
return fmt.Errorf("invalid health.storage_probe_interval %q: must be non-negative", h.StorageProbeInterval)
}
return nil
}
// Validate checks Gradle build cache settings, applying the default upload
// size if unset.
func (g *GradleBuildCacheConfig) Validate() error {
if g.MaxUploadSize == "" {
g.MaxUploadSize = defaultGradleMaxUploadSizeStr
}
uploadSize, err := ParseSize(g.MaxUploadSize)
if err != nil {
return fmt.Errorf("invalid gradle.build_cache.max_upload_size: %w", err)
}
if uploadSize <= 0 {
return fmt.Errorf("invalid gradle.build_cache.max_upload_size %q: must be > 0", g.MaxUploadSize)
}
if g.MaxAge != "" && g.MaxAge != "0" {
if _, err := parseDayDuration(g.MaxAge); err != nil {
return fmt.Errorf("invalid gradle.build_cache.max_age %q: %w", g.MaxAge, err)
}
}
if g.MaxSize != "" {
if _, err := ParseSize(g.MaxSize); err != nil {
return fmt.Errorf("invalid gradle.build_cache.max_size: %w", err)
}
}
if g.SweepInterval != "" {
d, err := time.ParseDuration(g.SweepInterval)
if err != nil {
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: %w", g.SweepInterval, err)
}
if d <= 0 {
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: must be > 0", g.SweepInterval)
}
}
return nil
}
const (
defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default
defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default
defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default
defaultHitFlushInterval = time.Second
defaultMetadataMaxSize = 100 << 20
defaultMetadataRewriteCacheSize = 256 << 20
defaultGradleBuildCacheMaxUploadSize = 100 << 20
defaultGradleBuildCacheSweepInterval = 10 * time.Minute
defaultGradleMaxUploadSizeStr = "100MB"
defaultGradleSweepIntervalStr = "10m"
defaultRetentionSweepInterval = 10 * time.Minute
defaultRetentionSweepIntervalStr = "10m"
minRetentionSweepInterval = time.Minute
maxDurationDays = 36500
defaultScanningTimeoutStr = "30s"
)
// ParseMaxSize returns the maximum cache size in bytes.
// Returns 0 if unset or explicitly disabled (meaning unlimited).
func (c *Config) ParseMaxSize() int64 {
if c.Storage.MaxSize == "" || c.Storage.MaxSize == "0" {
return 0
}
size, err := ParseSize(c.Storage.MaxSize)
if err != nil {
return 0
}
return size
}
func validateHTTPTimeout(s string) error {
if s == "" || s == "0" {
return nil
}
d, err := time.ParseDuration(s)
if err != nil {
return fmt.Errorf("invalid http_timeout %q: %w", s, err)
}
if d < 0 {
return fmt.Errorf("invalid http_timeout %q: must be non-negative", s)
}
return nil
}
func validateMetadataMaxSize(s string) error {
if s == "" {
return nil
}
size, err := ParseSize(s)
if err != nil {
return fmt.Errorf("invalid metadata_max_size: %w", err)
}
if size <= 0 {
return fmt.Errorf("invalid metadata_max_size %q: must be positive", s)
}
return nil
}
func (c *Config) validateDownloadSizeLimits() error {
if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil {
return err
}
return validateArtifactMaxSize(c.ArtifactMaxSize)
}
func validateArtifactMaxSize(s string) error {
if s == "" {
return nil
}
size, err := ParseSize(s)
if err != nil {
return fmt.Errorf("invalid artifact_max_size: %w", err)
}
if size < 0 {
return fmt.Errorf("invalid artifact_max_size %q: must not be negative", s)
}
return nil
}
func validateMetadataRewriteCacheSize(s string) error {
if s == "" || s == "0" {
return nil
}
size, err := ParseSize(s)
if err != nil {
return fmt.Errorf("invalid metadata_rewrite_cache_size: %w", err)
}
if size < 0 {
return fmt.Errorf("invalid metadata_rewrite_cache_size %q: must not be negative", s)
}
return nil
}
// ParseMetadataRewriteCacheSize returns how many bytes of rewritten metadata
// to keep in memory. Returns 256MB if unset or invalid, 0 if disabled.
func (c *Config) ParseMetadataRewriteCacheSize() int64 {
if c.MetadataRewriteCacheSize == "" {
return defaultMetadataRewriteCacheSize
}
if c.MetadataRewriteCacheSize == "0" {
return 0
}
size, err := ParseSize(c.MetadataRewriteCacheSize)
if err != nil || size < 0 {
return defaultMetadataRewriteCacheSize
}
return size
}
// ParseMetadataMaxSize returns the maximum metadata response size in bytes.
// Returns 100MB if unset or invalid.
func (c *Config) ParseMetadataMaxSize() int64 {
if c.MetadataMaxSize == "" {
return defaultMetadataMaxSize
}
size, err := ParseSize(c.MetadataMaxSize)
if err != nil || size <= 0 {
return defaultMetadataMaxSize
}
return size
}
// ParseArtifactMaxSize returns the artifact size limit in bytes, or 0 for
// no limit.
func (c *Config) ParseArtifactMaxSize() int64 {
if c.ArtifactMaxSize == "" {
return 0
}
size, err := ParseSize(c.ArtifactMaxSize)
if err != nil || size < 0 {
return 0
}
return size
}
// ParseHTTPTimeout returns the upstream HTTP client timeout.
// Returns 30s if unset, 0 (no timeout) if explicitly set to "0".
func (c *Config) ParseHTTPTimeout() time.Duration {
if c.HTTPTimeout == "" {
return defaultHTTPTimeout
}
if c.HTTPTimeout == "0" {
return 0
}
d, err := time.ParseDuration(c.HTTPTimeout)
if err != nil || d < 0 {
return defaultHTTPTimeout
}
return d
}
func validateHitFlushInterval(s string) error {
if s == "" || s == "0" {
return nil
}
d, err := time.ParseDuration(s)
if err != nil {
return fmt.Errorf("invalid database.hit_flush_interval %q: %w", s, err)
}
if d < 0 {
return fmt.Errorf("invalid database.hit_flush_interval %q: must be non-negative", s)
}
return nil
}
// ParseHitFlushInterval returns how often batched cache hits are written.
// Returns 1 second if unset or invalid, 0 if explicitly disabled.
func (c *Config) ParseHitFlushInterval() time.Duration {
if c.Database.HitFlushInterval == "" {
return defaultHitFlushInterval
}
if c.Database.HitFlushInterval == "0" {
return 0
}
d, err := time.ParseDuration(c.Database.HitFlushInterval)
if err != nil || d < 0 {
return defaultHitFlushInterval
}
return d
}
// ParseMetadataTTL returns the metadata TTL duration.
// Returns 5 minutes if unset, 0 if explicitly disabled.
func (c *Config) ParseMetadataTTL() time.Duration {
if c.MetadataTTL == "" {
return defaultMetadataTTL
}
if c.MetadataTTL == "0" {
return 0
}
d, err := time.ParseDuration(c.MetadataTTL)
if err != nil {
return defaultMetadataTTL
}
return d
}
// ParseGradleBuildCacheMaxUploadSize returns the max accepted PUT body size.
// Defaults to 100MB if unset or invalid.
func (c *Config) ParseGradleBuildCacheMaxUploadSize() int64 {
if c.Gradle.BuildCache.MaxUploadSize == "" {
return defaultGradleBuildCacheMaxUploadSize
}
size, err := ParseSize(c.Gradle.BuildCache.MaxUploadSize)
if err != nil || size <= 0 {
return defaultGradleBuildCacheMaxUploadSize
}
return size
}
// ParseGradleBuildCacheMaxAge returns age-based eviction threshold.
// Returns 0 when disabled or invalid.
func (c *Config) ParseGradleBuildCacheMaxAge() time.Duration {
if c.Gradle.BuildCache.MaxAge == "" || c.Gradle.BuildCache.MaxAge == "0" {
return 0
}
d, err := parseDayDuration(c.Gradle.BuildCache.MaxAge)
if err != nil || d <= 0 {
return 0
}
return d
}
// ParseGradleBuildCacheMaxSize returns total-size cap in bytes.
// Returns 0 when disabled or invalid.
func (c *Config) ParseGradleBuildCacheMaxSize() int64 {
if c.Gradle.BuildCache.MaxSize == "" || c.Gradle.BuildCache.MaxSize == "0" {
return 0
}
size, err := ParseSize(c.Gradle.BuildCache.MaxSize)
if err != nil || size <= 0 {
return 0
}
return size
}
// ParseGradleBuildCacheSweepInterval returns eviction sweep cadence.
// Defaults to 10m if unset or invalid.
func (c *Config) ParseGradleBuildCacheSweepInterval() time.Duration {
if c.Gradle.BuildCache.SweepInterval == "" {
return defaultGradleBuildCacheSweepInterval
}
d, err := time.ParseDuration(c.Gradle.BuildCache.SweepInterval)
if err != nil || d <= 0 {
return defaultGradleBuildCacheSweepInterval
}
return d
}
// ParseDirectServeTTL returns the presigned URL expiry duration.
// Returns 15 minutes if unset.
func (c *Config) ParseDirectServeTTL() time.Duration {
if c.Storage.DirectServeTTL == "" {
return defaultDirectServeTTL
}
d, err := time.ParseDuration(c.Storage.DirectServeTTL)
if err != nil {
return defaultDirectServeTTL
}
return d
}
// ParseSize parses a human-readable size string (e.g., "10GB", "500MB").
// Returns the size in bytes.
func ParseSize(s string) (int64, error) {
s = strings.TrimSpace(strings.ToUpper(s))
if s == "" || s == "0" {
return 0, nil
}
// Check suffixes in order of length (longest first) to avoid partial matches
suffixes := []struct {
suffix string
mult int64
}{
{"TB", 1024 * 1024 * 1024 * 1024},
{"GB", 1024 * 1024 * 1024},
{"MB", 1024 * 1024},
{"KB", 1024},
{"T", 1024 * 1024 * 1024 * 1024},
{"G", 1024 * 1024 * 1024},
{"M", 1024 * 1024},
{"K", 1024},
{"B", 1},
}
for _, s2 := range suffixes {
if numStr, ok := strings.CutSuffix(s, s2.suffix); ok {
num, err := strconv.ParseFloat(numStr, 64)
if err != nil {
return 0, fmt.Errorf("invalid number %q", numStr)
}
return int64(num * float64(s2.mult)), nil
}
}
// Try parsing as plain number (bytes)
num, err := strconv.ParseInt(s, 10, 64)
if err != nil {
return 0, fmt.Errorf("invalid size %q", s)
}
return num, nil
}
// Header returns the HTTP header name and value for this auth config.
// Returns empty strings if the config is invalid or incomplete.
func (a *AuthConfig) Header() (name, value string) {
switch strings.ToLower(a.Type) {
case "bearer":
token := expandEnv(a.Token)
if token == "" {
return "", ""
}
return "Authorization", "Bearer " + token
case "basic":
username := expandEnv(a.Username)
password := expandEnv(a.Password)
if username == "" {
return "", ""
}
encoded := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
return "Authorization", "Basic " + encoded
case "header":
name := a.HeaderName
value := expandEnv(a.HeaderValue)
if name == "" {
return "", ""
}
return name, value
default:
return "", ""
}
}
// expandEnv expands ${VAR_NAME} references in a string.
func expandEnv(s string) string {
return os.Expand(s, os.Getenv)
}
func envBool(v string) bool {
return v == "true" || v == "1"
}