Make metadata size limit configurable (closes #149) (#150)

Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 7.2.1 to 7.2.2.
- [Release notes](https://github.com/goreleaser/goreleaser-action/releases)
- [Commits](1a80836c5c...5daf1e915a)

---
updated-dependencies:
- dependency-name: goreleaser/goreleaser-action
  dependency-version: 7.2.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/publish.yml

@@ -38,7 +38,7 @@ jobs:
           images: ghcr.io/${{ github.repository }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
+        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf
         with:
           context: .
           push: true

.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
           go-version-file: go.mod
           cache: false
 
-      - uses: goreleaser/goreleaser-action@1a80836c5c9d9e5755a25cb59ec6f45a3b5f41a8 # v7.2.1
+      - uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2
         with:
           version: "~> v2"
           args: release --clean

.github/workflows/zizmor.yml

@@ -26,4 +26,4 @@ jobs:
           persist-credentials: false
 
       - name: Run zizmor
-        uses: zizmorcore/zizmor-action@a16621b09c6db4281f81a93cb393b05dcd7b7165 # v0.5.5
+        uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6

cmd/proxy/main.go

@@ -470,6 +470,7 @@ func runMirror() {
 	proxy := handler.NewProxy(db, store, fetcher, resolver, logger)
 	proxy.CacheMetadata = true // mirror always caches metadata
 	proxy.MetadataTTL = cfg.ParseMetadataTTL()
+	proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
 
 	m := mirror.New(proxy, db, store, logger, *concurrency)
 

docs/configuration.md

@@ -265,6 +265,16 @@ Set to `"0"` to always revalidate with upstream (ETag-based conditional requests
 
 When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated.
 
+### Metadata size limit
+
+Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs.
+
+```yaml
+metadata_max_size: "100MB"   # default
+```
+
+Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`.
+
 ## Mirror API
 
 The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP:

go.mod

@@ -7,10 +7,10 @@ require (
 	github.com/CycloneDX/cyclonedx-go v0.11.0
 	github.com/git-pkgs/archives v0.3.0
 	github.com/git-pkgs/cooldown v0.1.1
-	github.com/git-pkgs/enrichment v0.2.2
+	github.com/git-pkgs/enrichment v0.2.3
 	github.com/git-pkgs/purl v0.1.12
-	github.com/git-pkgs/registries v0.6.0
+	github.com/git-pkgs/registries v0.6.1
-	github.com/git-pkgs/spdx v0.1.3
+	github.com/git-pkgs/spdx v0.1.4
 	github.com/git-pkgs/vers v0.2.6
 	github.com/git-pkgs/vulns v0.1.5
 	github.com/go-chi/chi/v5 v5.2.5
@@ -129,7 +129,7 @@ require (
 	github.com/ghostiam/protogetter v0.3.20 // indirect
 	github.com/git-pkgs/packageurl-go v0.3.1 // indirect
 	github.com/git-pkgs/pom v0.1.4 // indirect
-	github.com/github/go-spdx/v2 v2.6.0 // indirect
+	github.com/github/go-spdx/v2 v2.7.0 // indirect
 	github.com/go-critic/go-critic v0.14.3 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

go.sum

@@ -254,24 +254,24 @@ github.com/git-pkgs/archives v0.3.0 h1:iXKyO83jEFub1PGEDlHmk2tQ7XeV5LySTc0sEkH3x
 github.com/git-pkgs/archives v0.3.0/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU=
 github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w=
 github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E=
-github.com/git-pkgs/enrichment v0.2.2 h1:vaQu5vs3tjQB5JI0gzBrUCynUc9z3l5byPhgKFaNZrc=
+github.com/git-pkgs/enrichment v0.2.3 h1:42mqoUhQZNGhlEO671pboI/Cu6F+DoffJoFbVhb2jlw=
-github.com/git-pkgs/enrichment v0.2.2/go.mod h1:5JWGmlHWcv5HQHUrctcpnRUNpEF5VAixD2z4zvqKejs=
+github.com/git-pkgs/enrichment v0.2.3/go.mod h1:MBv5nhHzjwLxeSgx2+7waCcpReUjhCD+9B0bvufpMO0=
 github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M=
 github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4=
 github.com/git-pkgs/pom v0.1.4 h1:C6st+XSbF75eKuwfdkDZZtYHoTcaWRIEQYar5VtszUo=
 github.com/git-pkgs/pom v0.1.4/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk=
 github.com/git-pkgs/purl v0.1.12 h1:qCskrEU1LWQhCkIVZd992W5++Bsxazvx2Cx1/65qCvU=
 github.com/git-pkgs/purl v0.1.12/go.mod h1:ofp4mHsR0cUeVONQaf33n6Wxg2QTEvtUdRfCedI8ouA=
-github.com/git-pkgs/registries v0.6.0 h1:ttQC8via9XAoLk9vqysf0K7uWl1bAyHPBWRBavRpAqs=
+github.com/git-pkgs/registries v0.6.1 h1:xZfVZQmffIfdeJthn5o2EozbVJ6gBeImYwKQnfdKUfU=
-github.com/git-pkgs/registries v0.6.0/go.mod h1:BY0YW+V0WDGBMuDR2aSMR3NzOPFK4K+F3j6+ch+cq3M=
+github.com/git-pkgs/registries v0.6.1/go.mod h1:a3BP/56VW3O/CFRqiJCtSy+OqRrSH25wF1PWHP76ka0=
-github.com/git-pkgs/spdx v0.1.3 h1:YQou23mLfzbW//6JlHUuc5x1P5VNIIDSku5gvauf86I=
+github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs=
-github.com/git-pkgs/spdx v0.1.3/go.mod h1:4HGGWyC8tg4DjOhrtBTYl4Lu+5i2BFuauGX8zcVcYPg=
+github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto=
 github.com/git-pkgs/vers v0.2.6 h1:IelZd7BP/JhzTloUTDY67nehUgoYva3g9viqAMCHJg8=
 github.com/git-pkgs/vers v0.2.6/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo=
 github.com/git-pkgs/vulns v0.1.5 h1:mtX88/27toFl+B95kaH5QbAdOCQ3YIDGjJrlrrnqQTE=
 github.com/git-pkgs/vulns v0.1.5/go.mod h1:bZFikfrR/5gC0ZMwXh7qcEu2gpKfXMBhVsy4kF12Ae0=
-github.com/github/go-spdx/v2 v2.6.0 h1:Y/Chr7L8oG85Ilbzl11xkUSQFUfG1kGkLP18LyInvhg=
+github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ=
-github.com/github/go-spdx/v2 v2.6.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0=
+github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0=
 github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
 github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
 github.com/go-critic/go-critic v0.14.3 h1:5R1qH2iFeo4I/RJU8vTezdqs08Egi4u5p6vOESA0pog=

internal/config/config.go

@@ -96,6 +96,11 @@ type Config struct {
 	// Default: "5m". Set to "0" to always revalidate.
 	MetadataTTL string `json:"metadata_ttl" yaml:"metadata_ttl"`
 
+	// MetadataMaxSize is the maximum size of an upstream metadata response
+	// the proxy will buffer (e.g. "100MB", "250MB"). Responses over this
+	// size return ErrMetadataTooLarge. Default: "100MB".
+	MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"`
+
 	// MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP.
 	// Disabled by default to prevent unauthenticated users from triggering downloads.
 	MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"`
@@ -424,6 +429,9 @@ func (c *Config) LoadFromEnv() {
 	if v := os.Getenv("PROXY_METADATA_TTL"); v != "" {
 		c.MetadataTTL = v
 	}
+	if v := os.Getenv("PROXY_METADATA_MAX_SIZE"); v != "" {
+		c.MetadataMaxSize = v
+	}
 	if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY"); v != "" {
 		c.Gradle.BuildCache.ReadOnly = v == "true" || v == "1"
 	}
@@ -513,6 +521,10 @@ func (c *Config) Validate() error {
 		}
 	}
 
+	if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil {
+		return err
+	}
+
 	if err := c.Health.Validate(); err != nil {
 		return err
 	}
@@ -582,6 +594,7 @@ func (g *GradleBuildCacheConfig) Validate() error {
 const (
 	defaultMetadataTTL                   = 5 * time.Minute  //nolint:mnd // sensible default
 	defaultDirectServeTTL                = 15 * time.Minute //nolint:mnd // sensible default
+	defaultMetadataMaxSize               = 100 << 20
 	defaultGradleBuildCacheMaxUploadSize = 100 << 20
 	defaultGradleBuildCacheSweepInterval = 10 * time.Minute
 	defaultGradleMaxUploadSizeStr        = "100MB"
@@ -601,6 +614,33 @@ func (c *Config) ParseMaxSize() int64 {
 	return size
 }
 
+func validateMetadataMaxSize(s string) error {
+	if s == "" {
+		return nil
+	}
+	size, err := ParseSize(s)
+	if err != nil {
+		return fmt.Errorf("invalid metadata_max_size: %w", err)
+	}
+	if size <= 0 {
+		return fmt.Errorf("invalid metadata_max_size %q: must be positive", s)
+	}
+	return nil
+}
+
+// ParseMetadataMaxSize returns the maximum metadata response size in bytes.
+// Returns 100MB if unset or invalid.
+func (c *Config) ParseMetadataMaxSize() int64 {
+	if c.MetadataMaxSize == "" {
+		return defaultMetadataMaxSize
+	}
+	size, err := ParseSize(c.MetadataMaxSize)
+	if err != nil || size <= 0 {
+		return defaultMetadataMaxSize
+	}
+	return size
+}
+
 // ParseMetadataTTL returns the metadata TTL duration.
 // Returns 5 minutes if unset, 0 if explicitly disabled.
 func (c *Config) ParseMetadataTTL() time.Duration {

internal/config/config_test.go

@@ -428,6 +428,52 @@ func TestParseMetadataTTL(t *testing.T) {
 	}
 }
 
+func TestParseMetadataMaxSize(t *testing.T) {
+	tests := []struct {
+		name string
+		size string
+		want int64
+	}{
+		{"unset uses default", "", defaultMetadataMaxSize},
+		{"explicit value", "250MB", 250 << 20},
+		{"bytes", "1024", 1024},
+		{"invalid uses default", "lots", defaultMetadataMaxSize},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := Default()
+			cfg.MetadataMaxSize = tt.size
+			got := cfg.ParseMetadataMaxSize()
+			if got != tt.want {
+				t.Errorf("ParseMetadataMaxSize() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestValidateMetadataMaxSize(t *testing.T) {
+	cfg := Default()
+	cfg.MetadataMaxSize = "not-a-size"
+	if err := cfg.Validate(); err == nil {
+		t.Error("expected validation error for invalid metadata_max_size")
+	}
+
+	cfg.MetadataMaxSize = "0"
+	if err := cfg.Validate(); err == nil {
+		t.Error("expected validation error for zero metadata_max_size")
+	}
+
+	cfg.MetadataMaxSize = "250MB"
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("unexpected error for valid metadata_max_size: %v", err)
+	}
+
+	cfg.MetadataMaxSize = ""
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("unexpected error for unset metadata_max_size: %v", err)
+	}
+}
+
 func TestValidateMetadataTTL(t *testing.T) {
 	cfg := Default()
 	cfg.MetadataTTL = "invalid"

internal/handler/conda.go

@@ -161,7 +161,7 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	body, err := ReadMetadata(resp.Body)
+	body, err := h.proxy.ReadMetadata(resp.Body)
 	if err != nil {
 		http.Error(w, "failed to read response", http.StatusInternalServerError)
 		return

internal/handler/handler.go

@@ -52,23 +52,25 @@ const contentTypeJSON = "application/json"
 
 const headerAcceptEncoding = "Accept-Encoding"
 
-// maxMetadataSize is the maximum size of upstream metadata responses (100 MB).
+// defaultMetadataMaxSize is used when Proxy.MetadataMaxSize is unset.
-// Package metadata (e.g. npm with many versions) can be large, but unbounded
+const defaultMetadataMaxSize = 100 << 20
-// reads risk OOM if an upstream misbehaves.
-const maxMetadataSize = 100 << 20
 
-// ErrMetadataTooLarge is returned when upstream metadata exceeds maxMetadataSize.
+// ErrMetadataTooLarge is returned when upstream metadata exceeds the configured limit.
 var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit")
 
 // ReadMetadata reads an upstream response body with a size limit to prevent OOM
 // from unexpectedly large responses. Returns ErrMetadataTooLarge if the response
 // is truncated by the limit.
-func ReadMetadata(r io.Reader) ([]byte, error) {
+func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) {
-	data, err := io.ReadAll(io.LimitReader(r, maxMetadataSize+1))
+	limit := p.MetadataMaxSize
+	if limit <= 0 {
+		limit = defaultMetadataMaxSize
+	}
+	data, err := io.ReadAll(io.LimitReader(r, limit+1))
 	if err != nil {
 		return nil, err
 	}
-	if int64(len(data)) > maxMetadataSize {
+	if int64(len(data)) > limit {
 		return nil, ErrMetadataTooLarge
 	}
 	return data, nil
@@ -84,6 +86,7 @@ type Proxy struct {
 	Cooldown            *cooldown.Config
 	CacheMetadata       bool
 	MetadataTTL         time.Duration
+	MetadataMaxSize     int64
 	GradleReadOnly      bool
 	GradleMaxUploadSize int64
 	DirectServe         bool
@@ -474,7 +477,7 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u
 			cached, readErr := p.Storage.Open(ctx, entry.StoragePath)
 			if readErr == nil {
 				defer func() { _ = cached.Close() }()
-				data, readErr := ReadMetadata(cached)
+				data, readErr := p.ReadMetadata(cached)
 				if readErr == nil {
 					ct := contentTypeJSON
 					if entry.ContentType.Valid {
@@ -519,7 +522,7 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u
 	}
 	defer func() { _ = cached.Close() }()
 
-	data, readErr := ReadMetadata(cached)
+	data, readErr := p.ReadMetadata(cached)
 	if readErr != nil {
 		return nil, "", fmt.Errorf("upstream failed and cached read error: %w", err)
 	}
@@ -561,7 +564,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
 			return nil, "", "", zeroTime, errStale304
 		}
 		defer func() { _ = cached.Close() }()
-		data, readErr := ReadMetadata(cached)
+		data, readErr := p.ReadMetadata(cached)
 		if readErr != nil {
 			return nil, "", "", zeroTime, errStale304
 		}
@@ -583,7 +586,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
 		return nil, "", "", zeroTime, fmt.Errorf("upstream returned %d", resp.StatusCode)
 	}
 
-	body, err := ReadMetadata(resp.Body)
+	body, err := p.ReadMetadata(resp.Body)
 	if err != nil {
 		return nil, "", "", zeroTime, fmt.Errorf("reading response: %w", err)
 	}

internal/handler/nuget.go

@@ -193,7 +193,7 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	body, err := ReadMetadata(resp.Body)
+	body, err := h.proxy.ReadMetadata(resp.Body)
 	if err != nil {
 		http.Error(w, "failed to read response", http.StatusInternalServerError)
 		return

internal/handler/read_metadata_test.go

@@ -7,9 +7,12 @@ import (
 )
 
 func TestReadMetadata(t *testing.T) {
+	const limit = 1024
+	p := &Proxy{MetadataMaxSize: limit}
+
 	t.Run("small body", func(t *testing.T) {
 		data := []byte("hello world")
-		got, err := ReadMetadata(bytes.NewReader(data))
+		got, err := p.ReadMetadata(bytes.NewReader(data))
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
@@ -19,27 +22,39 @@ func TestReadMetadata(t *testing.T) {
 	})
 
 	t.Run("exactly at limit", func(t *testing.T) {
-		data := make([]byte, maxMetadataSize)
+		data := make([]byte, limit)
 		for i := range data {
 			data[i] = 'x'
 		}
-		got, err := ReadMetadata(bytes.NewReader(data))
+		got, err := p.ReadMetadata(bytes.NewReader(data))
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
-		if len(got) != int(maxMetadataSize) {
+		if len(got) != limit {
-			t.Errorf("got length %d, want %d", len(got), maxMetadataSize)
+			t.Errorf("got length %d, want %d", len(got), limit)
 		}
 	})
 
 	t.Run("over limit returns error", func(t *testing.T) {
-		data := make([]byte, maxMetadataSize+100)
+		data := make([]byte, limit+100)
 		for i := range data {
 			data[i] = 'x'
 		}
-		_, err := ReadMetadata(bytes.NewReader(data))
+		_, err := p.ReadMetadata(bytes.NewReader(data))
 		if !errors.Is(err, ErrMetadataTooLarge) {
 			t.Errorf("got error %v, want ErrMetadataTooLarge", err)
 		}
 	})
+
+	t.Run("zero limit uses default", func(t *testing.T) {
+		p := &Proxy{}
+		data := make([]byte, 1<<20)
+		got, err := p.ReadMetadata(bytes.NewReader(data))
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if len(got) != len(data) {
+			t.Errorf("got length %d, want %d", len(got), len(data))
+		}
+	})
 }

internal/server/server.go

@@ -160,6 +160,7 @@ func (s *Server) Start() error {
 	proxy.Cooldown = cd
 	proxy.CacheMetadata = s.cfg.CacheMetadata
 	proxy.MetadataTTL = s.cfg.ParseMetadataTTL()
+	proxy.MetadataMaxSize = s.cfg.ParseMetadataMaxSize()
 	proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly
 	proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize()
 	proxy.DirectServe = s.cfg.Storage.DirectServe